\[ \int x^4 \left (a+b x^2\right )^{5/2} \left (A+B x^2\right ) \, dx \]
Optimal antiderivative \[ \frac {a \left (12 A b -5 a B \right ) x^{5} \left (b \,x^{2}+a \right )^{\frac {3}{2}}}{192 b}+\frac {\left (12 A b -5 a B \right ) x^{5} \left (b \,x^{2}+a \right )^{\frac {5}{2}}}{120 b}+\frac {B \,x^{5} \left (b \,x^{2}+a \right )^{\frac {7}{2}}}{12 b}+\frac {a^{5} \left (12 A b -5 a B \right ) \arctanh \! \left (\frac {x \sqrt {b}}{\sqrt {b \,x^{2}+a}}\right )}{1024 b^{\frac {7}{2}}}-\frac {a^{4} \left (12 A b -5 a B \right ) x \sqrt {b \,x^{2}+a}}{1024 b^{3}}+\frac {a^{3} \left (12 A b -5 a B \right ) x^{3} \sqrt {b \,x^{2}+a}}{1536 b^{2}}+\frac {a^{2} \left (12 A b -5 a B \right ) x^{5} \sqrt {b \,x^{2}+a}}{384 b} \]
command
integrate(x**4*(b*x**2+a)**(5/2)*(B*x**2+A),x)
Sympy 1.10.1 under Python 3.10.4 output
\[ \text {Timed out} \]
Sympy 1.8 under Python 3.8.8 output
\[ - \frac {3 A a^{\frac {9}{2}} x}{256 b^{2} \sqrt {1 + \frac {b x^{2}}{a}}} - \frac {A a^{\frac {7}{2}} x^{3}}{256 b \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {129 A a^{\frac {5}{2}} x^{5}}{640 \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {73 A a^{\frac {3}{2}} b x^{7}}{160 \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {29 A \sqrt {a} b^{2} x^{9}}{80 \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {3 A a^{5} \operatorname {asinh}{\left (\frac {\sqrt {b} x}{\sqrt {a}} \right )}}{256 b^{\frac {5}{2}}} + \frac {A b^{3} x^{11}}{10 \sqrt {a} \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {5 B a^{\frac {11}{2}} x}{1024 b^{3} \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {5 B a^{\frac {9}{2}} x^{3}}{3072 b^{2} \sqrt {1 + \frac {b x^{2}}{a}}} - \frac {B a^{\frac {7}{2}} x^{5}}{1536 b \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {55 B a^{\frac {5}{2}} x^{7}}{384 \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {67 B a^{\frac {3}{2}} b x^{9}}{192 \sqrt {1 + \frac {b x^{2}}{a}}} + \frac {7 B \sqrt {a} b^{2} x^{11}}{24 \sqrt {1 + \frac {b x^{2}}{a}}} - \frac {5 B a^{6} \operatorname {asinh}{\left (\frac {\sqrt {b} x}{\sqrt {a}} \right )}}{1024 b^{\frac {7}{2}}} + \frac {B b^{3} x^{13}}{12 \sqrt {a} \sqrt {1 + \frac {b x^{2}}{a}}} \]