Let $$I=\int_0^1\frac{\ln^2x\ln(1+x)}{1+x^2}\ dx$$
and $$I(a)=\int_0^1\frac{\ln^2x\ln(1+ax)}{1+x^2}\ dx$$
$$I(0)=0 , \quad I(1)=I$$
\begin{align}
I^{'}(a)&=\int_0^1\frac{x\ln^2x}{(1+ax)(1+x^2)}\ dx=\int_0^1\frac{\ln^2x}{1+a^2}\left(\frac{a}{1+x^2}+\frac{x}{1+x^2}-\frac{a}{1+ax}\right)\ dx\\
&=\frac{a}{1+a^2}\int_0^1\frac{\ln^2x}{1+x^2}\ dx+\frac1{1+a^2}\int_0^1\frac{x\ln^2x}{1+x^2}\ dx-\frac{a}{1+a^2}\int_0^1\frac{\ln^2x}{1+ax}\ dx\\
&=\frac{a}{1+a^2}\left(\frac{\pi^3}{16}\right)+\frac1{1+a^2}\left(\frac{3}{16}\zeta(3)\right)-\frac{a}{1+a^2}\left(\frac{-2\operatorname{Li}_3(-a)}{a}\right)
\end{align}
Then
\begin{align}
I&=\frac{\pi^3}{16}\int_0^1\frac{a\ da}{1+a^2}+\frac{3}{16}\zeta(3)\int_0^1\frac{da}{1+a^2}+2\int_0^1\frac{\operatorname{Li}_3(-a)}{1+a^2}\ da\\
&=\frac{\pi^3}{32}\ln 2+\frac{3\pi}{64}\zeta(3)+2\color{blue}{\int_0^1\frac{\operatorname{Li}_3(-a)}{1+a^2}\ da}\tag{1}\\
\end{align}
.
To evaluate the blue integral, I am going to use the same approach in my post here:
\begin{align}
\int_0^1 \frac{\operatorname{Li}_3(-a)}{1+a^2}\ da&=\int_0^\infty \frac{\operatorname{Li}_3(-a)}{1+a^2}\ da-\underbrace{\int_1^\infty \frac{\operatorname{Li}_3(-a)}{1+a^2}\ da}_{a\mapsto 1/a}\\
&=\int_0^\infty \frac{\operatorname{Li}_3(-a)}{1+a^2}\ da-\int_0^1 \frac{\operatorname{Li}_3(-1/a)}{1+a^2}\ da\\
&\left\{\text{add the integral to both sides}\right\}\\
2\int_0^1 \frac{\operatorname{Li}_3(-a)}{1+a^2}\ da&=\int_0^\infty\frac{\operatorname{Li}_3(-a)}{1+a^2}\ da-\int_0^1 \frac{\operatorname{Li}_3(-1/a)-\operatorname{Li}_3(-a)}{1+a^2}\ da\\
&\{\text{use}\ \operatorname{Li}_3(-1/a)-\operatorname{Li}_3(-a)=\frac16\ln^3a+\zeta(2)\ln a\}\\
&=\int_0^\infty\frac{\operatorname{Li}_3(-a)}{1+a^2}\ da-\frac16\underbrace{\int_0^1\frac{\ln^3a}{1+a^2}\ da}_{-6\beta(4)}-\zeta(2)\underbrace{\int_0^1\frac{\ln a}{1+a^2}\ da}_{-G}\\
&=\color{red}{\int_0^\infty\frac{\operatorname{Li}_3(-a)}{1+a^2}\ da}+\beta(4)+\zeta(2)G\tag{2}
\end{align}
Also the red integral can be evaluated the same way in this solution:
$$\operatorname{Li}_{3}(-a)=\frac12\int_0^1\frac{-a\ln^2 u}{1+au}\ du$$
\begin{align}
\int_0^\infty\frac{\operatorname{Li}_{3}(-a)}{1+a^2}\ da&=-\frac12\int_0^1\ln^2u\left(\int_0^\infty\frac{a}{(1+ua)(1+a^2)}\ da\right)\ du\\
&=-\frac12\int_0^1\ln^2u\left(\frac12\left(\frac{\pi u}{1+u^2}-\frac{2\ln u}{1+u^2}\right)\right)\ du\\
&=-\frac{\pi}{4}\underbrace{\int_0^1\frac{u\ln^2u}{1+u^2}\ du}_{\frac{3}{16}\zeta(3)}+\frac12\underbrace{\int_0^1\frac{\ln^3u}{1+u^2}\ du}_{-6\beta(4)}\\
&=\frac{-3\pi}{64}\zeta(3)-3\beta(4)\tag{3}
\end{align}
Plugging (3) in (2) we get
$$2\int_0^1 \frac{\operatorname{Li}_3(-a)}{1+a^2}\ da=\zeta(2)G-\frac{3\pi}{64}\zeta(3)-2\beta(4)\tag{4}$$
Now plug (4) in (1) we get
$$I=\int_0^1\frac{\ln^2x\ln(1+x)}{1+x^2}\ dx=\frac{\pi^3}{32}\ln2+\zeta(2)G-2\beta(4)$$