\[ \int \frac {\coth ^3(x)}{\sqrt {a+b \tanh ^2(x)}} \, dx \]
Optimal antiderivative \[ -\frac {\left (2 a -b \right ) \arctanh \left (\frac {\sqrt {a +b \left (\tanh ^{2}\left (x \right )\right )}}{\sqrt {a}}\right )}{2 a^{\frac {3}{2}}}+\frac {\arctanh \left (\frac {\sqrt {a +b \left (\tanh ^{2}\left (x \right )\right )}}{\sqrt {a +b}}\right )}{\sqrt {a +b}}-\frac {\left (\coth ^{2}\left (x \right )\right ) \sqrt {a +b \left (\tanh ^{2}\left (x \right )\right )}}{2 a} \]
command
integrate(coth(x)^3/(a+b*tanh(x)^2)^(1/2),x, algorithm="giac")
Giac 1.9.0-11 via sagemath 9.6 output
\[ \frac {{\left (2 \, a - b\right )} \arctan \left (-\frac {\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b} - \sqrt {a + b}}{2 \, \sqrt {-a}}\right )}{\sqrt {-a} a} - \frac {\log \left ({\left | -{\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b}\right )} {\left (a + b\right )} - \sqrt {a + b} {\left (a - b\right )} \right |}\right )}{2 \, \sqrt {a + b}} + \frac {\log \left ({\left | -\sqrt {a + b} e^{\left (2 \, x\right )} + \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b} + \sqrt {a + b} \right |}\right )}{2 \, \sqrt {a + b}} - \frac {\log \left ({\left | -\sqrt {a + b} e^{\left (2 \, x\right )} + \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b} - \sqrt {a + b} \right |}\right )}{2 \, \sqrt {a + b}} + \frac {2 \, {\left ({\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b}\right )}^{3} {\left (2 \, a + b\right )} + {\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b}\right )}^{2} {\left (2 \, a - 3 \, b\right )} \sqrt {a + b} - {\left (2 \, a^{2} + 3 \, a b - 3 \, b^{2}\right )} {\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b}\right )} - {\left (2 \, a^{2} - a b + b^{2}\right )} \sqrt {a + b}\right )}}{{\left ({\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b}\right )}^{2} - 2 \, {\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} + 2 \, a e^{\left (2 \, x\right )} - 2 \, b e^{\left (2 \, x\right )} + a + b}\right )} \sqrt {a + b} - 3 \, a + b\right )}^{2} a} \]
Giac 1.7.0 via sagemath 9.3 output
\[ \text {Exception raised: TypeError} \]________________________________________________________________________________________