From 5abfdab18a8f594a5e5101f85bf3b8f2f8cf3f1d Mon Sep 17 00:00:00 2001 From: Claudio Ardagna Date: Thu, 13 Jun 2024 16:17:37 +0200 Subject: [PATCH] exp modificati --- experiment.tex | 151 +++++++++++++++++++++++++------------------------ 1 file changed, 76 insertions(+), 75 deletions(-) diff --git a/experiment.tex b/experiment.tex index 28729fe..8acc056 100644 --- a/experiment.tex +++ b/experiment.tex @@ -107,7 +107,7 @@ \subsection{Testing Infrastructure and Experimental Settings}\label{subsec:exper \label{fig:execution_example} \end{figure} - \subsection{Perfomance}\label{subsec:experiments_performance} + \subsection{Performance}\label{subsec:experiments_performance} We first measured the performance (execution time) of our exhaustive and heuristic solutions by varying the number of vertices in the pipeline template from 2 to 7 and the number of services per vertex from 2 to 7. \cref{fig:time_window_perce_average} presents our results for both the exhaustive and heuristic solutions. The exhaustive approach is able to provide the optimal solution for all configurations, but its execution time grows exponentially with the number of vertices and services, making it impractical for large instances. For \windowsize from 1 to 3 (step 1), we observed a substantial reduction in execution time, with the heuristic always able to produce an instance in less than $\approx2.7\times10^5ms$ . The worst heuristic performance (7 vertices, 7 services, \windowsize=6) is $\approx3.8\times10^7ms$ is still one order of magnitude lower than the best exhaustive performance (7 vertices, 7 services, \windowsize=7) $\approx1.35\times10^8ms$. \begin{figure}[!t] @@ -154,6 +154,78 @@ \subsection{Testing Infrastructure and Experimental Settings}\label{subsec:exper When considering setting \wide, the greedy approach (\windowsize=1) provides good results on average (0.71, 0.90), while showing substantial quality oscillations in specific runs: between 0.882 and 0.970 for 3 vertices, 0.810 and 0.942 for 4 vertices, 0.580 and 0.853 for 5 vertices, 0.682 and 0.943 for 6 vertices, 0.596 and 0.821 for 7 vertices. This same trend emerges when the window size is $<$$l$/2, while it starts approaching the optimum when the window size is $\geq$$l$/2. For instance, when \windowsize=$l$-1, the quality varies between 0.957 and 1.0 for 3 vertices, 0.982 and 1.0 for 4 vertices, 0.986 and 0.998 for 5 vertices, 0.977 and 1.0 for 6 vertices, 0.996 and 1.0 for 7 vertices. + \begin{figure}[H] + \centering + \begin{adjustbox}{minipage=\linewidth,scale=0.75} + \begin{subfigure}{0.45\textwidth} + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n3} + \caption{\wide 3 vertices} + \label{fig:quality_window_wide_perce_n3} + \end{subfigure} + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n4} + \caption{\wide 4 vertices} + \label{fig:quality_window_wide_perce_n4} + \end{subfigure} + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n5} + \caption{\wide 5 vertices} + + \label{fig:quality_window_wide_perce_n5} + \end{subfigure} + + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n6} + \caption{\wide 6 vertices} + \label{fig:quality_window_wide_perce_n6} + \end{subfigure} + + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n7} + \caption{\wide 7 vertices} + \label{fig:quality_window_wide_perce_n7} + \end{subfigure} + \end{subfigure} + \begin{subfigure}{0.45\textwidth} + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n3} + \caption{\average 3 vertices} + + \label{fig:quality_window_average_perce_n3} + \end{subfigure} + + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n4} + \caption{\average 4 vertices} + + \label{fig:quality_window_average_perce_n4} + \end{subfigure} + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n5} + \caption{\average 5 vertices} + \label{fig:quality_window_average_perce_n5} + \end{subfigure} + + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n6} + \caption{\average 6 vertices} + \label{fig:quality_window_average_perce_n6} + \end{subfigure} + + + \begin{subfigure}{\textwidth} + \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n7} + \caption{\average 7 vertices} + \label{fig:quality_window_average_perce_n7} + \end{subfigure} + + \end{subfigure} + + \caption{Evaluation of Quality Using the \emph{Qualitative} Metric in a \wide (\cref{fig:quality_window_wide_perce_n3,fig:quality_window_wide_perce_n4,fig:quality_window_wide_perce_n5,fig:quality_window_wide_perce_n6,fig:quality_window_wide_perce_n7}) and \average (\cref{fig:quality_window_average_perce_n3,fig:quality_window_average_perce_n4,fig:quality_window_average_perce_n5,fig:quality_window_average_perce_n6,fig:quality_window_average_perce_n7}) Profile Configuration.} \label{fig:quality_window_perce} + \end{adjustbox} + \end{figure} + When considering setting \average, the heuristic algorithm still provides good results, limiting the quality oscillations observed for setting \wide\ and approaching the quality of the exhaustive also for lower window sizes. The greedy approach (\windowsize=1) provides good results on average (from 0.842 to 0.944), as well as in specific runs: between 0.927 and 0.978 for 3 vertices, 0.903 and 0.962 for 4 vertices, 0.840 and 0.915 for 5 vertices, 0.815 and 0.934 for 6 vertices, 0.721 and 0.935 for 7 vertices. When \windowsize=$l$-1, the quality varies between 0.980 and 1.0 for 3 vertices, 0.978 and 1.0 for 4 vertices, 0.954 and 1 for 5 vertices, 0.987 and 1.0 for 6 vertices, 0.990 and 1.0 for 7 vertices. @@ -162,9 +234,6 @@ \subsection{Testing Infrastructure and Experimental Settings}\label{subsec:exper When considering setting \wide, the greedy approach (\windowsize=1) provides good results on average (0.92, 0.97), limiting oscillations observed with metric $M_J$; for instance, the quality varies between 0.951 and 0.989 for 3 vertices, 0.941 and 0.988 for 4 vertices, 0.919 and 0.974 for 5 vertices, 0.911 and 0.971 for 6 vertices, 0.877 and 0.924 for 7 vertices. The worst quality results are obtained with window size equal to 1, while the oscillations are negligible when the window size is >2. For instance, when \windowsize=$l$-2, the quality varies between, 0.982 and 0.996 for 4 vertices, 0.981 and 0.998 for 5 vertices, 0.988 and 1.0 for 6 vertices, 0.976 and 0.999 for 7 vertices. When \windowsize=$l$-1, the quality varies between 0.987 and 0.998 for 3 vertices, 0.993 and 1.0 for 4 vertices, 0.985 and 0.999 for 5 vertices, 0.997 and 1.0 for 6 vertices, 0.995 and 1.0 for 7 vertices. - When considering setting \average, the greedy approach (\windowsize=1) provides results similar to setting \wide. On average, quality varies from 0.920 to 0.969, limiting oscillations; for instance, the quality varies between 0.951 and 0.989 for 3 vertices, 0.942 and 0.988 for 4 vertices, 0.919 and 0.975 for 5 vertices, 0.912 and 0.972 for 6 vertices, 0.878 and 0.925 for 7 vertices. The \average configuration provides even tighter quality oscillations than the \wide configuration. Notably, the poorest quality outcomes are observed when the window size is set to 1. Conversely, these oscillations become negligible when the window size exceeds 1 in configurations with three and four vertices, and when it exceeds 2 in configurations involving five, six, and seven vertices. For instance, when \windowsize=3, the quality varies between 0.993 and 1 for 4 vertices, 0.981 and 0.998 for 5 vertices, 0.982 and 997 for 6 vertices, 0.960 and 0.991 for 7 vertices. - - Our results suggest that the proposed heuristic well approximates the results obtained by the exhaustive approach. While larger window sizes generally lead to better performance, there exists a breakpoint where the balance between window size and performance is optimized. Beyond this point, the incremental gains in metric values may not justify the additional computational burden or complexity introduced by larger windows. It is worth noting that lower window sizes are more unstable, especially with setting \wide, meaning that the quality varies significantly among different configurations. This effect stabilizes with higher window sizes (e.g., \windowsize$\geq$$l$/2). \begin{figure}[H] \centering \begin{adjustbox}{minipage=\linewidth,scale=0.75} @@ -234,78 +303,10 @@ \subsection{Testing Infrastructure and Experimental Settings}\label{subsec:exper \end{adjustbox} \end{figure} + When considering setting \average, the greedy approach (\windowsize=1) provides results similar to setting \wide. On average, quality varies from 0.920 to 0.969, limiting oscillations; for instance, the quality varies between 0.951 and 0.989 for 3 vertices, 0.942 and 0.988 for 4 vertices, 0.919 and 0.975 for 5 vertices, 0.912 and 0.972 for 6 vertices, 0.878 and 0.925 for 7 vertices. The \average configuration provides even tighter quality oscillations than the \wide configuration. Notably, the poorest quality outcomes are observed when the window size is set to 1. Conversely, these oscillations become negligible when the window size exceeds 1 in configurations with three and four vertices, and when it exceeds 2 in configurations involving five, six, and seven vertices. For instance, when \windowsize=3, the quality varies between 0.993 and 1 for 4 vertices, 0.981 and 0.998 for 5 vertices, 0.982 and 997 for 6 vertices, 0.960 and 0.991 for 7 vertices. + Our results suggest that the proposed heuristic well approximates the results obtained by the exhaustive approach. While larger window sizes generally lead to better performance, there exists a breakpoint where the balance between window size and performance is optimized. Beyond this point, the incremental gains in metric values may not justify the additional computational burden or complexity introduced by larger windows. It is worth noting that lower window sizes are more unstable, especially with setting \wide, meaning that the quality varies significantly among different configurations. This effect stabilizes with higher window sizes (e.g., \windowsize$\geq$$l$/2). + -\begin{figure}[H] - \centering - \begin{adjustbox}{minipage=\linewidth,scale=0.75} - \begin{subfigure}{0.45\textwidth} - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n3} - \caption{\wide 3 vertices} - \label{fig:quality_window_wide_perce_n3} - \end{subfigure} - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n4} - \caption{\wide 4 vertices} - \label{fig:quality_window_wide_perce_n4} - \end{subfigure} - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n5} - \caption{\wide 5 vertices} - - \label{fig:quality_window_wide_perce_n5} - \end{subfigure} - - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n6} - \caption{\wide 6 vertices} - \label{fig:quality_window_wide_perce_n6} - \end{subfigure} - - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_20_100_n7} - \caption{\wide 7 vertices} - \label{fig:quality_window_wide_perce_n7} - \end{subfigure} -\end{subfigure} -\begin{subfigure}{0.45\textwidth} - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n3} - \caption{\average 3 vertices} - - \label{fig:quality_window_average_perce_n3} - \end{subfigure} - - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n4} - \caption{\average 4 vertices} - - \label{fig:quality_window_average_perce_n4} - \end{subfigure} - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n5} - \caption{\average 5 vertices} - \label{fig:quality_window_average_perce_n5} - \end{subfigure} - - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n6} - \caption{\average 6 vertices} - \label{fig:quality_window_average_perce_n6} - \end{subfigure} - - - \begin{subfigure}{\textwidth} - \includegraphics[width=\textwidth]{Images/graphs/window_quality_performance_diff_perce_n7_s7_50_89_n7} - \caption{\average 7 vertices} - \label{fig:quality_window_average_perce_n7} - \end{subfigure} - -\end{subfigure} - - \caption{Evaluation of Quality Using the \emph{Qualitative} Metric in a \wide (\cref{fig:quality_window_wide_perce_n3,fig:quality_window_wide_perce_n4,fig:quality_window_wide_perce_n5,fig:quality_window_wide_perce_n6,fig:quality_window_wide_perce_n7}) and \average (\cref{fig:quality_window_average_perce_n3,fig:quality_window_average_perce_n4,fig:quality_window_average_perce_n5,fig:quality_window_average_perce_n6,fig:quality_window_average_perce_n7}) Profile Configuration.} \label{fig:quality_window_perce} -\end{adjustbox} -\end{figure}