\[ \int \frac {-a^2 b+a (2 a+b) x-3 a x^2+x^3}{x (-b+x) \sqrt {x (-a+x) (-b+x)} \left (a-(1+b d) x+d x^2\right )} \, dx \]
Optimal antiderivative \[ \frac {2 \sqrt {a b x +\left (-a -b \right ) x^{2}+x^{3}}}{x \left (-b +x \right )}+2 \sqrt {d}\, \operatorname {arctanh}\left (\frac {\sqrt {d}\, \sqrt {a b x +\left (-a -b \right ) x^{2}+x^{3}}}{a -x}\right ) \]
command
Int[(-(a^2*b) + a*(2*a + b)*x - 3*a*x^2 + x^3)/(x*(-b + x)*Sqrt[x*(-a + x)*(-b + x)]*(a - (1 + b*d)*x + d*x^2)),x]
Rubi 4.17.3 under Mathematica 13.3.1 output
\[ \frac {2 \left (b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{b}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right )|\frac {b}{a}\right ) (a-x)}{\sqrt {b} \left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{a}}}+\frac {2 \left (b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{b}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right )|\frac {b}{a}\right ) (a-x)}{\sqrt {b} \left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{a}}}-\frac {4 \sqrt {x} \sqrt {1-\frac {x}{b}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right )|\frac {b}{a}\right ) (a-x)}{\sqrt {b} \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{a}}}-\frac {4 (b-x) (a-x)}{b \sqrt {(a-x) (b-x) x}}-\frac {4 x (a-x)}{b \sqrt {(a-x) (b-x) x}}+\frac {2 (a-x)}{\sqrt {(a-x) (b-x) x}}-\frac {4 \sqrt {a} d (b-x) \sqrt {x} \sqrt {1-\frac {x}{a}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right )|\frac {a}{b}\right )}{\left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{b}}}-\frac {4 \sqrt {a} d (b-x) \sqrt {x} \sqrt {1-\frac {x}{a}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right )|\frac {a}{b}\right )}{\left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{b}}}+\frac {2 \sqrt {b} \left (-2 a d+b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x}}+\frac {2 \sqrt {b} \left (-2 a d+b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x}}-\frac {4 a \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\sqrt {b} \sqrt {(a-x) (b-x) x}}+\frac {2 (2 a-b) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\sqrt {b} \sqrt {(a-x) (b-x) x}}-\frac {\sqrt {a} \left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticPi}\left (\frac {2 a d}{b d-\sqrt {(b d+1)^2-4 a d}+1},\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right ),\frac {a}{b}\right )}{\sqrt {(a-x) (b-x) x}}-\frac {\sqrt {a} \left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticPi}\left (\frac {2 a d}{b d+\sqrt {(b d+1)^2-4 a d}+1},\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right ),\frac {a}{b}\right )}{\sqrt {(a-x) (b-x) x}} \]
Rubi 4.16.1 under Mathematica 13.3.1 output
\[ \int \frac {-a^2 b+a (2 a+b) x-3 a x^2+x^3}{x (-b+x) \sqrt {x (-a+x) (-b+x)} \left (a-(1+b d) x+d x^2\right )} \, dx \]________________________________________________________________________________________