\[ \int \left (a+b \coth ^2(x)\right )^{3/2} \tanh ^2(x) \, dx \]
Optimal antiderivative \[ -b^{\frac {3}{2}} \arctanh \! \left (\frac {\coth \! \left (x \right ) \sqrt {b}}{\sqrt {a +b \left (\coth ^{2}\left (x \right )\right )}}\right )+\left (a +b \right )^{\frac {3}{2}} \arctanh \! \left (\frac {\coth \! \left (x \right ) \sqrt {a +b}}{\sqrt {a +b \left (\coth ^{2}\left (x \right )\right )}}\right )-a \sqrt {a +b \left (\coth ^{2}\left (x \right )\right )}\, \tanh \! \left (x \right ) \]
command
integrate((a+b*coth(x)^2)^(3/2)*tanh(x)^2,x, algorithm="giac")
Giac 1.9.0-11 via sagemath 9.6 output
\[ \text {could not integrate} \]
Giac 1.7.0 via sagemath 9.3 output
\[ \frac {2 \, b^{2} \arctan \left (-\frac {\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} - 2 \, a e^{\left (2 \, x\right )} + 2 \, b e^{\left (2 \, x\right )} + a + b} - \sqrt {a + b}}{2 \, \sqrt {-b}}\right ) \mathrm {sgn}\left (e^{\left (2 \, x\right )} - 1\right )}{\sqrt {-b}} - \frac {1}{2} \, {\left (a + b\right )}^{\frac {3}{2}} \log \left ({\left | \sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} - 2 \, a e^{\left (2 \, x\right )} + 2 \, b e^{\left (2 \, x\right )} + a + b} + \sqrt {a + b} \right |}\right ) \mathrm {sgn}\left (e^{\left (2 \, x\right )} - 1\right ) + \frac {1}{2} \, {\left (a + b\right )}^{\frac {3}{2}} \log \left ({\left | -\sqrt {a + b} e^{\left (2 \, x\right )} + \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} - 2 \, a e^{\left (2 \, x\right )} + 2 \, b e^{\left (2 \, x\right )} + a + b} + \sqrt {a + b} \right |}\right ) \mathrm {sgn}\left (e^{\left (2 \, x\right )} - 1\right ) - \frac {{\left (a^{2} + 2 \, a b + b^{2}\right )} \log \left ({\left | -{\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} - 2 \, a e^{\left (2 \, x\right )} + 2 \, b e^{\left (2 \, x\right )} + a + b}\right )} {\left (a + b\right )} + \sqrt {a + b} {\left (a - b\right )} \right |}\right ) \mathrm {sgn}\left (e^{\left (2 \, x\right )} - 1\right )}{2 \, \sqrt {a + b}} - \frac {4 \, {\left ({\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} - 2 \, a e^{\left (2 \, x\right )} + 2 \, b e^{\left (2 \, x\right )} + a + b}\right )} a^{2} \mathrm {sgn}\left (e^{\left (2 \, x\right )} - 1\right ) - \sqrt {a + b} a^{2} \mathrm {sgn}\left (e^{\left (2 \, x\right )} - 1\right )\right )}}{{\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} - 2 \, a e^{\left (2 \, x\right )} + 2 \, b e^{\left (2 \, x\right )} + a + b}\right )}^{2} + 2 \, {\left (\sqrt {a + b} e^{\left (2 \, x\right )} - \sqrt {a e^{\left (4 \, x\right )} + b e^{\left (4 \, x\right )} - 2 \, a e^{\left (2 \, x\right )} + 2 \, b e^{\left (2 \, x\right )} + a + b}\right )} \sqrt {a + b} - 3 \, a + b} \]