\[ \int \left (\pi +c^2 \pi x^2\right )^{5/2} \left (a+b \sinh ^{-1}(c x)\right )^2 \, dx \]
Optimal antiderivative \[ \frac {65 b^{2} \pi ^{\frac {5}{2}} x \left (c^{2} x^{2}+1\right )^{\frac {3}{2}}}{1728}+\frac {b^{2} \pi ^{\frac {5}{2}} x \left (c^{2} x^{2}+1\right )^{\frac {5}{2}}}{108}-\frac {115 b^{2} \pi ^{\frac {5}{2}} \arcsinh \left (c x \right )}{1152 c}-\frac {5 b c \,\pi ^{\frac {5}{2}} x^{2} \left (a +b \arcsinh \left (c x \right )\right )}{16}-\frac {5 b \,\pi ^{\frac {5}{2}} \left (c^{2} x^{2}+1\right )^{2} \left (a +b \arcsinh \left (c x \right )\right )}{48 c}-\frac {b \,\pi ^{\frac {5}{2}} \left (c^{2} x^{2}+1\right )^{3} \left (a +b \arcsinh \left (c x \right )\right )}{18 c}+\frac {5 \pi x \left (c^{2} \pi \,x^{2}+\pi \right )^{\frac {3}{2}} \left (a +b \arcsinh \left (c x \right )\right )^{2}}{24}+\frac {x \left (c^{2} \pi \,x^{2}+\pi \right )^{\frac {5}{2}} \left (a +b \arcsinh \left (c x \right )\right )^{2}}{6}+\frac {5 \pi ^{\frac {5}{2}} \left (a +b \arcsinh \left (c x \right )\right )^{3}}{48 b c}+\frac {245 b^{2} \pi ^{\frac {5}{2}} x \sqrt {c^{2} x^{2}+1}}{1152}+\frac {5 \pi ^{2} x \left (a +b \arcsinh \left (c x \right )\right )^{2} \sqrt {c^{2} \pi \,x^{2}+\pi }}{16} \]
command
integrate((pi*c**2*x**2+pi)**(5/2)*(a+b*asinh(c*x))**2,x)
Sympy 1.10.1 under Python 3.10.4 output
\[ \begin {cases} \frac {\pi ^{\frac {5}{2}} a^{2} c^{4} x^{5} \sqrt {c^{2} x^{2} + 1}}{6} + \frac {13 \pi ^{\frac {5}{2}} a^{2} c^{2} x^{3} \sqrt {c^{2} x^{2} + 1}}{24} + \frac {11 \pi ^{\frac {5}{2}} a^{2} x \sqrt {c^{2} x^{2} + 1}}{16} + \frac {5 \pi ^{\frac {5}{2}} a^{2} \operatorname {asinh}{\left (c x \right )}}{16 c} - \frac {\pi ^{\frac {5}{2}} a b c^{5} x^{6}}{18} + \frac {\pi ^{\frac {5}{2}} a b c^{4} x^{5} \sqrt {c^{2} x^{2} + 1} \operatorname {asinh}{\left (c x \right )}}{3} - \frac {13 \pi ^{\frac {5}{2}} a b c^{3} x^{4}}{48} + \frac {13 \pi ^{\frac {5}{2}} a b c^{2} x^{3} \sqrt {c^{2} x^{2} + 1} \operatorname {asinh}{\left (c x \right )}}{12} - \frac {11 \pi ^{\frac {5}{2}} a b c x^{2}}{16} + \frac {11 \pi ^{\frac {5}{2}} a b x \sqrt {c^{2} x^{2} + 1} \operatorname {asinh}{\left (c x \right )}}{8} + \frac {5 \pi ^{\frac {5}{2}} a b \operatorname {asinh}^{2}{\left (c x \right )}}{16 c} - \frac {\pi ^{\frac {5}{2}} b^{2} c^{5} x^{6} \operatorname {asinh}{\left (c x \right )}}{18} + \frac {\pi ^{\frac {5}{2}} b^{2} c^{4} x^{5} \sqrt {c^{2} x^{2} + 1} \operatorname {asinh}^{2}{\left (c x \right )}}{6} + \frac {\pi ^{\frac {5}{2}} b^{2} c^{4} x^{5} \sqrt {c^{2} x^{2} + 1}}{108} - \frac {13 \pi ^{\frac {5}{2}} b^{2} c^{3} x^{4} \operatorname {asinh}{\left (c x \right )}}{48} + \frac {13 \pi ^{\frac {5}{2}} b^{2} c^{2} x^{3} \sqrt {c^{2} x^{2} + 1} \operatorname {asinh}^{2}{\left (c x \right )}}{24} + \frac {97 \pi ^{\frac {5}{2}} b^{2} c^{2} x^{3} \sqrt {c^{2} x^{2} + 1}}{1728} - \frac {11 \pi ^{\frac {5}{2}} b^{2} c x^{2} \operatorname {asinh}{\left (c x \right )}}{16} + \frac {11 \pi ^{\frac {5}{2}} b^{2} x \sqrt {c^{2} x^{2} + 1} \operatorname {asinh}^{2}{\left (c x \right )}}{16} + \frac {299 \pi ^{\frac {5}{2}} b^{2} x \sqrt {c^{2} x^{2} + 1}}{1152} + \frac {5 \pi ^{\frac {5}{2}} b^{2} \operatorname {asinh}^{3}{\left (c x \right )}}{48 c} - \frac {299 \pi ^{\frac {5}{2}} b^{2} \operatorname {asinh}{\left (c x \right )}}{1152 c} & \text {for}\: c \neq 0 \\\pi ^{\frac {5}{2}} a^{2} x & \text {otherwise} \end {cases} \]
Sympy 1.8 under Python 3.8.8 output
\[ \text {Timed out} \]________________________________________________________________________________________