\[ \int \frac {\coth (x)}{\left (a+b \tanh ^2(x)\right )^{5/2}} \, dx \]
Optimal antiderivative \[ -\frac {\arctanh \left (\frac {\sqrt {a +b \left (\tanh ^{2}\left (x \right )\right )}}{\sqrt {a}}\right )}{a^{\frac {5}{2}}}+\frac {\arctanh \left (\frac {\sqrt {a +b \left (\tanh ^{2}\left (x \right )\right )}}{\sqrt {a +b}}\right )}{\left (a +b \right )^{\frac {5}{2}}}+\frac {b \left (2 a +b \right )}{a^{2} \left (a +b \right )^{2} \sqrt {a +b \left (\tanh ^{2}\left (x \right )\right )}}+\frac {b}{3 a \left (a +b \right ) \left (a +b \left (\tanh ^{2}\left (x \right )\right )\right )^{\frac {3}{2}}} \]
command
integrate(coth(x)/(a+b*tanh(x)^2)^(5/2),x, algorithm="giac")
Giac 1.9.0-11 via sagemath 9.6 output
\[ -\frac {\sqrt {a + b} \log \left ({\left | -{\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b}\right )} {\left (a + b\right )} - \sqrt {a + b} {\left (a - b\right )} \right |}\right )}{2 \, {\left (a^{3} + 3 \, a^{2} b + 3 \, a b^{2} + b^{3}\right )}} + \frac {\sqrt {a + b} \log \left ({\left | -\sqrt {a + b} e^{\left (2 \, x\right )} + \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b} + \sqrt {a + b} \right |}\right )}{2 \, {\left (a^{3} + 3 \, a^{2} b + 3 \, a b^{2} + b^{3}\right )}} - \frac {\sqrt {a + b} \log \left ({\left | -\sqrt {a + b} e^{\left (2 \, x\right )} + \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b} - \sqrt {a + b} \right |}\right )}{2 \, {\left (a^{3} + 3 \, a^{2} b + 3 \, a b^{2} + b^{3}\right )}} + \frac {{\left ({\left (\frac {{\left (7 \, a^{14} b^{3} + 38 \, a^{13} b^{4} + 85 \, a^{12} b^{5} + 100 \, a^{11} b^{6} + 65 \, a^{10} b^{7} + 22 \, a^{9} b^{8} + 3 \, a^{8} b^{9}\right )} e^{\left (2 \, x\right )}}{a^{16} b^{2} + 6 \, a^{15} b^{3} + 15 \, a^{14} b^{4} + 20 \, a^{13} b^{5} + 15 \, a^{12} b^{6} + 6 \, a^{11} b^{7} + a^{10} b^{8}} + \frac {3 \, {\left (7 \, a^{14} b^{3} + 30 \, a^{13} b^{4} + 49 \, a^{12} b^{5} + 36 \, a^{11} b^{6} + 9 \, a^{10} b^{7} - 2 \, a^{9} b^{8} - a^{8} b^{9}\right )}}{a^{16} b^{2} + 6 \, a^{15} b^{3} + 15 \, a^{14} b^{4} + 20 \, a^{13} b^{5} + 15 \, a^{12} b^{6} + 6 \, a^{11} b^{7} + a^{10} b^{8}}\right )} e^{\left (2 \, x\right )} + \frac {3 \, {\left (7 \, a^{14} b^{3} + 30 \, a^{13} b^{4} + 49 \, a^{12} b^{5} + 36 \, a^{11} b^{6} + 9 \, a^{10} b^{7} - 2 \, a^{9} b^{8} - a^{8} b^{9}\right )}}{a^{16} b^{2} + 6 \, a^{15} b^{3} + 15 \, a^{14} b^{4} + 20 \, a^{13} b^{5} + 15 \, a^{12} b^{6} + 6 \, a^{11} b^{7} + a^{10} b^{8}}\right )} e^{\left (2 \, x\right )} + \frac {7 \, a^{14} b^{3} + 38 \, a^{13} b^{4} + 85 \, a^{12} b^{5} + 100 \, a^{11} b^{6} + 65 \, a^{10} b^{7} + 22 \, a^{9} b^{8} + 3 \, a^{8} b^{9}}{a^{16} b^{2} + 6 \, a^{15} b^{3} + 15 \, a^{14} b^{4} + 20 \, a^{13} b^{5} + 15 \, a^{12} b^{6} + 6 \, a^{11} b^{7} + a^{10} b^{8}}}{3 \, {\left (a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b\right )}^{\frac {3}{2}}} + \frac {2 \, \arctan \left (-\frac {\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b} - \sqrt {a + b}}{2 \, \sqrt {-a}}\right )}{\sqrt {-a} a^{2}} \]
Giac 1.7.0 via sagemath 9.3 output
\[ \text {Exception raised: TypeError} \]________________________________________________________________________________________