;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; This file is part of "NOVA": NOVA = search + COCOMO tools ; Copyright, 2008, Tim Menzies tim@menzies.us ; ; NOVA is free software: you can redistribute it and/or modify ; it under the terms of the GNU General Public License as published by ; the Free Software Foundation, either version 3 of the License, or ; (at your option) any later version. ; ; NOVA is distributed in the hope that it will be useful, ; but WITHOUT ANY WARRANTY; without even the implied warranty of ; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ; GNU General Public License for more details. ; You should have received a copy of the GNU General Public License ; a long with NOVA. If not, see . ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (in-package :xomo.experiments) (defun results-compare-create-report-pdf (experiment-directory names search-engine-id policy-id case-study-id) (with-open-file (fstream (merge-pathnames "energy-vs-risk-exposure-results-compare.tex" experiment-directory) :direction :output :if-exists :supersede :if-does-not-exist :create) (macrolet ((out (control-string &rest args) `(format fstream ,control-string ,@args))) (out "~&\\documentclass[letterpaper,10pt]{article}") (out "~&\\usepackage{graphicx}") (out "~&% pagelayout") (out "~&%header") (out "~&\\setlength{\\topmargin}{-0.5in}") (out "~&\\setlength{\\headheight}{0in}") (out "~&\\setlength{\\headsep}{0in}") (out "~&%side margin") (out "~&\\setlength{\\oddsidemargin}{0in}") (out "~&\\setlength{\\evensidemargin}{0in}") (out "~&% text size") (out "~&\\setlength{\\textwidth}{6.5in}") (out "~&\\setlength{\\textheight}{9in}") (out "~&% footer") (out "~&\\setlength{\\footskip}{.25in}") (out "~&% define the title") (out "~&\\title{Energy vs Risk Exposure - Results Compare}") (out "~&\\begin{document}") (out "~&Compares normalized scores of Energy and Risk Exposure\\\\") (out "~&Search Engine: ~a\\\\" search-engine-id) (out "~&Policy: ~a\\\\" policy-id) (out "~&Case Study: ~a\\\\" case-study-id) (dolist (name names) (out "~&\\includegraphics[width=3.25in]{results-compare-plot-~a.png}" name)) (out "~&\\end{document}") (terpri fstream)))) (defun results-compare-create-run-all-script (experiment-directory names) ;;TODO pathname of compare report needs moved into a variable (with-open-file (fstream (merge-pathnames "run-all.sh" experiment-directory) :direction :output :if-exists :supersede :if-does-not-exist :create) (macrolet ((out (control-string &rest args) `(format fstream ,control-string ,@args))) (dolist (name names) (out "~&bash results-compare-plot-~a-gn.sh" name)) (out "~&pdflatex energy-vs-risk-exposure-results-compare.tex") (terpri fstream)))) (defun generate-results-compare-scores (results scoring-method-ids &key (runs 1000) static-scoring-method-ids) "generates an alist keys on score-fn-list. The data is a list of (decision-number median spread)" (let ((counter 1) scores) (dolist (result results) (let* ((projects (nova-simulate :runs runs :scoring-methods scoring-method-ids :constraints (constraints? (result-best-state result) t))) ;;calc med and spread from scores of simulated projects (run-scores (mapcar #'(lambda (scoring-method-id) (multiple-value-bind (median) (median (mapcar #'(lambda (project) (geta scoring-method-id project)) projects)) (cons scoring-method-id (list median)))) scoring-method-ids)) (static-scores (mapcar #'(lambda (static-scoring-method-id) (list static-scoring-method-id (funcall static-scoring-method-id result))) static-scoring-method-ids))) ;;save scores to run (mapc #'(lambda (scoring-method-id) (pusha scoring-method-id (cons counter (geta scoring-method-id run-scores)) scores)) scoring-method-ids) (mapc #'(lambda (static-scoring-method-id) (pusha static-scoring-method-id (cons counter (geta static-scoring-method-id static-scores)) scores)) static-scoring-method-ids)) (incf counter)) scores)) (defun results-compare-sort-n-normalize-scores (scores max-score) (let* ((tscores (transpose scores)) (indexes (first tscores)) (nvalues (mapcar #'(lambda (x) (coerce (/ x max-score) 'float)) (second tscores)))) (transpose (list (sort indexes #'<) (sort nvalues #'<))))) (defun create-constraints-report (output-directory filename constraints-list) (let ((total (* 1.0 (length constraints-list))) (features-ht (make-hash-table :test 'equalp)) (ranges-ht (make-hash-table :test 'equalp))) (labels ((inchash (key ht) (let ((c (gethash key ht 0))) (setf (gethash key ht) (1+ c))))) (dolist (constraints constraints-list) (dolist (constraint constraints) (let ((attribute (first constraint)) (value (second constraint))) (inchash attribute features-ht) (inchash (list attribute value) ranges-ht))))) (let (features-results ranges-results) (maphash #'(lambda (attribute count) (push (list attribute (/ count total)) features-results)) features-ht) (maphash #'(lambda (attr-val count) (push (list attr-val (/ count total)) ranges-results)) ranges-ht) (format t "~&features: ~{~a~}" features-results) (format t "~&ranges: ~{~a~}" ranges-results)))) (defun run-energy-vs-risk-exposure-results-compare-experiment (&key (search-engine-id 'nova.impl.search-engines:seesaw-r) (policy-id 'xomo.policies:all) (case-study-id 'nova.impl.case-studies:default) (find-best-state-method-id 'min-score-state-of-path-by-ttest?) (results 20) (runs 100) (base-output-directory nova.settings:*nova-output-dir*)) (with-new-db (init-db) (let* ((experiment-output-directory (merge-pathnames "energy-vs-risk-exposure-results-compare/" base-output-directory)) (score-name-fn-list (list (cons "effort" 'xomo.model:effort) (cons "months" 'xomo.model:months) (cons "defects" 'xomo.model:defects))) (static-score-name-fn-list (list (cons "decisions" 'result-num-features))) (search-engine (lookup-search-engine search-engine-id)) (policy (lookup-policy policy-id)) (case-study (lookup-case-study case-study-id)) (find-best-state-method (lookup-find-best-state-method find-best-state-method-id)) (learn-by-energy-results) (learn-by-risk-exposure-results)) (format t "~&Searching...") (dotimes (i results) (format t "~&~4d" (1+ i)) (format t "~tenergy...") (push (nova.apps.learning:nova-learn (lookup-evaluation-method 'xomo.evaluation-methods:energy) search-engine find-best-state-method :case-study case-study :policy policy) learn-by-energy-results) (format t "~trisk-exposure...") (push (nova.apps.learning:nova-learn (lookup-evaluation-method 'xomo.evaluation-methods:risk-exposure) search-engine find-best-state-method :case-study case-study :policy policy) learn-by-risk-exposure-results)) (ensure-directories-exist experiment-output-directory :verbose t) (format t "~&Processing...") (let ((learn-by-energy-results-compare-scores (generate-results-compare-scores learn-by-energy-results (mapcar #'cdr score-name-fn-list) :static-scoring-method-ids (mapcar #'cdr static-score-name-fn-list) :runs runs)) (learn-by-risk-exposure-results-compare-scores (generate-results-compare-scores learn-by-risk-exposure-results (mapcar #'cdr score-name-fn-list) :static-scoring-method-ids (mapcar #'cdr static-score-name-fn-list) :runs runs))) (dolist (score-name-fn (append score-name-fn-list static-score-name-fn-list)) (let* ((lbe-scores (geta (cdr score-name-fn) learn-by-energy-results-compare-scores)) (lbre-scores (geta (cdr score-name-fn) learn-by-risk-exposure-results-compare-scores)) (max-score (apply #'max (append (mapcar #'second lbe-scores) ;first is counter, second is mean (mapcar #'second lbre-scores)))) (snlbe-scores (results-compare-sort-n-normalize-scores lbe-scores max-score)) (snlbre-scores (results-compare-sort-n-normalize-scores lbre-scores max-score)) (xlabel (format nil "all ~a values, sorted" (car score-name-fn))) (ylabel (format nil "normalized ~a" (car score-name-fn))) (script-name (merge-pathnames (format nil "results-compare-plot-~a-gn.sh" (car score-name-fn)) experiment-output-directory)) (output-name (merge-pathnames (format nil "results-compare-plot-~a.png" (car score-name-fn)) experiment-output-directory))) (output-gnuplot-shell-script (list (create-line-plot-descriptor snlbe-scores 1 2 '((:title "Energy") (:style "linespoints"))) (create-line-plot-descriptor snlbre-scores 1 2 '((:title "Value") (:style "linespoints")))) script-name :modifiers `((:terminal "png") (:legend "top" "left") (:title ,(format nil "~a-~a" case-study-id policy-id)) (:yrange 0 1) (:ytic 0.2) (:xrange 1 ,results) (:xtic 1) (:xlabel ,xlabel) (:ylabel ,ylabel)) :gnuplot-redirect output-name)))) (create-constraints-report experiment-output-directory "energy.tex" (mapcar #'(lambda (result) (constraints? (result-best-state result))) learn-by-energy-results)) (create-constraints-report experiment-output-directory "risk-exposure.tex" (mapcar #'(lambda (result) (constraints? (result-best-state result))) learn-by-risk-exposure-results)) (results-compare-create-report-pdf experiment-output-directory (mapcar #'car (append score-name-fn-list static-score-name-fn-list)) search-engine-id policy-id case-study-id) (results-compare-create-run-all-script experiment-output-directory (mapcar #'car (append score-name-fn-list static-score-name-fn-list))) (format t "~&Complete~%") t)))