\[ \int \frac {\sinh ^2(x)}{a \cosh (x)+b \sinh (x)} \, dx \]
Optimal antiderivative \[ -\frac {a^{2} \arctan \left (\frac {b \cosh \left (x \right )+a \sinh \left (x \right )}{\sqrt {a^{2}-b^{2}}}\right )}{\left (a^{2}-b^{2}\right )^{\frac {3}{2}}}-\frac {b \cosh \left (x \right )}{a^{2}-b^{2}}+\frac {a \sinh \left (x \right )}{a^{2}-b^{2}} \]
command
integrate(sinh(x)**2/(a*cosh(x)+b*sinh(x)),x)
Sympy 1.10.1 under Python 3.10.4 output
\[ \begin {cases} \tilde {\infty } \cosh {\left (x \right )} & \text {for}\: a = 0 \wedge b = 0 \\\frac {\cosh {\left (x \right )}}{b} & \text {for}\: a = 0 \\- \frac {\sinh ^{2}{\left (x \right )}}{- 3 b \sinh {\left (x \right )} + 3 b \cosh {\left (x \right )}} - \frac {2 \sinh {\left (x \right )} \cosh {\left (x \right )}}{- 3 b \sinh {\left (x \right )} + 3 b \cosh {\left (x \right )}} + \frac {2 \cosh ^{2}{\left (x \right )}}{- 3 b \sinh {\left (x \right )} + 3 b \cosh {\left (x \right )}} & \text {for}\: a = - b \\- \frac {\sinh ^{2}{\left (x \right )}}{3 b \sinh {\left (x \right )} + 3 b \cosh {\left (x \right )}} + \frac {2 \sinh {\left (x \right )} \cosh {\left (x \right )}}{3 b \sinh {\left (x \right )} + 3 b \cosh {\left (x \right )}} + \frac {2 \cosh ^{2}{\left (x \right )}}{3 b \sinh {\left (x \right )} + 3 b \cosh {\left (x \right )}} & \text {for}\: a = b \\- \frac {a^{2} \log {\left (\tanh {\left (\frac {x}{2} \right )} + \frac {b}{a} - \frac {\sqrt {- a^{2} + b^{2}}}{a} \right )} \tanh ^{2}{\left (\frac {x}{2} \right )}}{a^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} - a^{2} \sqrt {- a^{2} + b^{2}} - b^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {- a^{2} + b^{2}}} + \frac {a^{2} \log {\left (\tanh {\left (\frac {x}{2} \right )} + \frac {b}{a} - \frac {\sqrt {- a^{2} + b^{2}}}{a} \right )}}{a^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} - a^{2} \sqrt {- a^{2} + b^{2}} - b^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {- a^{2} + b^{2}}} + \frac {a^{2} \log {\left (\tanh {\left (\frac {x}{2} \right )} + \frac {b}{a} + \frac {\sqrt {- a^{2} + b^{2}}}{a} \right )} \tanh ^{2}{\left (\frac {x}{2} \right )}}{a^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} - a^{2} \sqrt {- a^{2} + b^{2}} - b^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {- a^{2} + b^{2}}} - \frac {a^{2} \log {\left (\tanh {\left (\frac {x}{2} \right )} + \frac {b}{a} + \frac {\sqrt {- a^{2} + b^{2}}}{a} \right )}}{a^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} - a^{2} \sqrt {- a^{2} + b^{2}} - b^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {- a^{2} + b^{2}}} - \frac {2 a \sqrt {- a^{2} + b^{2}} \tanh {\left (\frac {x}{2} \right )}}{a^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} - a^{2} \sqrt {- a^{2} + b^{2}} - b^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {- a^{2} + b^{2}}} + \frac {2 b \sqrt {- a^{2} + b^{2}}}{a^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} - a^{2} \sqrt {- a^{2} + b^{2}} - b^{2} \sqrt {- a^{2} + b^{2}} \tanh ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {- a^{2} + b^{2}}} & \text {otherwise} \end {cases} \]
Sympy 1.8 under Python 3.8.8 output
\[ \text {Timed out} \]________________________________________________________________________________________