\[ \int \frac {a^2 b-a (2 a+b) x+3 a x^2-x^3}{x (-b+x) \sqrt {x (-a+x) (-b+x)} \left (a+(-1-b d) x+d x^2\right )} \, dx \]
Optimal antiderivative \[ \frac {2 \sqrt {a b x -a \,x^{2}-b \,x^{2}+x^{3}}}{\left (b -x \right ) x}+2 \sqrt {d}\, \operatorname {arctanh}\left (\frac {\sqrt {a b x +\left (-a -b \right ) x^{2}+x^{3}}}{\sqrt {d}\, x \left (-b +x \right )}\right ) \]
command
Int[(a^2*b - a*(2*a + b)*x + 3*a*x^2 - x^3)/(x*(-b + x)*Sqrt[x*(-a + x)*(-b + x)]*(a + (-1 - b*d)*x + d*x^2)),x]
Rubi 4.17.3 under Mathematica 13.3.1 output
\[ -\frac {2 \left (b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{b}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right )|\frac {b}{a}\right ) (a-x)}{\sqrt {b} \left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{a}}}-\frac {2 \left (b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{b}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right )|\frac {b}{a}\right ) (a-x)}{\sqrt {b} \left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{a}}}+\frac {4 \sqrt {x} \sqrt {1-\frac {x}{b}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right )|\frac {b}{a}\right ) (a-x)}{\sqrt {b} \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{a}}}+\frac {4 (b-x) (a-x)}{b \sqrt {(a-x) (b-x) x}}+\frac {4 x (a-x)}{b \sqrt {(a-x) (b-x) x}}-\frac {2 (a-x)}{\sqrt {(a-x) (b-x) x}}+\frac {4 \sqrt {a} d (b-x) \sqrt {x} \sqrt {1-\frac {x}{a}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right )|\frac {a}{b}\right )}{\left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{b}}}+\frac {4 \sqrt {a} d (b-x) \sqrt {x} \sqrt {1-\frac {x}{a}} E\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right )|\frac {a}{b}\right )}{\left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x} \sqrt {1-\frac {x}{b}}}-\frac {2 \sqrt {b} \left (-2 a d+b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x}}-\frac {2 \sqrt {b} \left (-2 a d+b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {(a-x) (b-x) x}}+\frac {4 a \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\sqrt {b} \sqrt {(a-x) (b-x) x}}-\frac {2 (2 a-b) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticF}\left (\arcsin \left (\frac {\sqrt {x}}{\sqrt {b}}\right ),\frac {b}{a}\right )}{\sqrt {b} \sqrt {(a-x) (b-x) x}}+\frac {\sqrt {a} \left (-b d-\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticPi}\left (\frac {2 a d}{b d-\sqrt {(b d+1)^2-4 a d}+1},\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right ),\frac {a}{b}\right )}{\sqrt {(a-x) (b-x) x}}+\frac {\sqrt {a} \left (-b d+\sqrt {(b d+1)^2-4 a d}+1\right ) \sqrt {x} \sqrt {1-\frac {x}{a}} \sqrt {1-\frac {x}{b}} \operatorname {EllipticPi}\left (\frac {2 a d}{b d+\sqrt {(b d+1)^2-4 a d}+1},\arcsin \left (\frac {\sqrt {x}}{\sqrt {a}}\right ),\frac {a}{b}\right )}{\sqrt {(a-x) (b-x) x}} \]
Rubi 4.16.1 under Mathematica 13.3.1 output
\[ \int \frac {a^2 b-a (2 a+b) x+3 a x^2-x^3}{x (-b+x) \sqrt {x (-a+x) (-b+x)} \left (a+(-1-b d) x+d x^2\right )} \, dx \]________________________________________________________________________________________