source: rtems-testing/rtems-coverage/do_coverage @ 9d00371

4.11
Last change on this file since 9d00371 was 9d00371, checked in by Joel Sherrill <joel.sherrill@…>, on Sep 28, 2009 at 8:47:48 PM

2009-09-28 Joel Sherrill <joel.sherrill@…>

  • do_coverage, run_coverage: Do not publish results by default.
  • Property mode set to 100755
File size: 13.4 KB
Line 
1#! /bin/sh
2#
3#  Script to perform a single coverage test run on a single BSP
4#  in a specific configuration.  This script automates all steps
5#  of building RTEMS, running the tests, and generating the reports.
6#
7#  $Id$
8#
9
10vfile=`dirname $0`/VERSIONS-COVERAGE
11if [ ! -r ${vfile} ] ; then
12  echo VERSIONS-COVERAGE file not found
13  exit 1
14fi
15
16source ${vfile}
17
18if [ ! -d ${BASEDIR} ] ; then
19  echo Have you set the BASEDIR in VERSIONS correctly?
20  exit 1
21fi
22
23progname=${0##*/}        # fast basename hack for ksh, bash
24
25USAGE=\
26"usage: $progname [ -opts ]
27        -v        -- verbose (default=no)
28        -A        -- Execute all steps (default=no)
29        -B BSP    -- Specify BSP to test
30        -P        -- Enable POSIX API (default=yes)
31        -D        -- Enable Developmental Coverage (default=no)
32        -S        -- Optimize for size (default=no)
33        -m        -- Update and build covMerge (default=no)
34        -u        -- Do CVS update on RTEMS (default=no)
35        -c        -- Configure RTEMS (default=no)
36        -b        -- Build RTEMS (default=no)
37        -C        -- copy tests from build to test execution point (default=no)
38        -r        -- run the tests (default=no)
39        -f        -- publish the results to ftp site (default=no)
40        -R        -- generate a report (default=no)
41
42Notes:  + There are currently NO checks at each step to determine if
43          the previous steps were performed!!!
44        + Use of each option toggles the setting.  For example, \"-v -v -A -u\"
45          results in verbose=no and all steps done except CVS update on RTEMS.
46
47"
48
49#   log an error to stderr
50prerr()
51{
52    echo "$*" >&2
53}
54
55fatal() {
56    prerr "$USAGE"
57    [ "$1" ] && (prerr ; prerr $*);
58    exit 1
59}
60
61warn() {
62    [ "$1" ] && prerr $*
63}
64
65check_status()
66{
67  if [ $1 -ne 0 ] ; then
68    shift
69    echo "FAILED: " "$*" >&2
70    exit 1
71  fi
72}
73
74toggle()
75{
76  case $1 in
77    no)  echo "yes" ;;
78    yes) echo "no" ;;
79    *)   fatal "Unknown value to toggle ($1)" ;;
80  esac
81}
82
83# parse arguments for these
84verbose="no"
85do_all="no"
86do_posix="yes"
87do_developmental="no"
88do_optimize_size="no"
89do_covmerge="no"
90do_rtems_update="no"
91do_rtems_configure="no"
92do_rtems_build="no"
93do_copy_tests="no"
94do_run_tests="no"
95do_report="no"
96do_publish="yes"
97BSP="not_set"
98
99while getopts vB:PDSmAucbCrRf OPT
100do
101    case "$OPT" in
102        A)
103          do_all="yes"
104          do_covmerge="yes"
105          do_posix="yes"
106          do_rtems_update="yes"
107          do_rtems_configure="yes"
108          do_rtems_build="yes"
109          do_copy_tests="yes"
110          do_run_tests="yes"
111          do_report="yes"
112          do_publish="yes"
113          ;;
114        B) BSP="${OPTARG}";;
115        v) verbose=`toggle ${verbose}` ;;
116        m) do_covmerge=`toggle ${do_covmerge}` ;;
117        P) do_posix=`toggle ${do_posix}` ;;
118        D) do_developmental=`toggle ${do_developmental}` ;;
119        S) do_optimize_size=`toggle ${do_optimize_size}` ;;
120        u) do_rtems_update=`toggle ${do_rtems_update}` ;;
121        c) do_rtems_configure=`toggle ${do_rtems_configure}` ;;
122        b) do_rtems_build=`toggle ${do_rtems_build}` ;;
123        C) do_copy_tests=`toggle ${do_copy_tests}` ;;
124        r) do_run_tests=`toggle ${do_run_tests}` ;;
125        R) do_report=`toggle ${do_report}` ;;
126        f) do_publish=`toggle ${do_publish}` ;;
127        *) fatal;;
128    esac
129done
130
131#  If we are to update or configure RTEMS, then we need to clean the
132#  RTEMS build tree up.
133do_clean="no"
134if [ ${do_rtems_update} = "yes" -o \
135     ${do_rtems_configure} = "yes" ] ; then
136  do_clean="yes"
137fi
138
139# allow bsp override
140if [ ${BSP} == "not_set" ] ; then
141  fatal "No BSP selected"
142fi
143
144# you can't publish unless ftp directory is accessible
145if [ ${do_publish} = "yes" ] ; then
146  if [ ! -d ${FTPDIR} ] ; then
147    fatal "Publishing results requested and ${FTPDIR} does not exist"
148  fi
149fi
150
151# For most BSPs, there is a script with the same name as the BSP.
152RUNNER=${BSP}
153
154case ${BSP} in
155  pc386)
156    CPU=i386
157    COVERAGE_FORMAT=RTEMS
158    RTEMS_EXTRA_ARGS="USE_COM1_AS_CONSOLE=1"
159    RTEMS_EXTRA_ARGS="${RTEMS_EXTRA_ARGS} BSP_PRESS_KEY_FOR_RESET=0"
160    ;;
161  uC5282)
162    CPU=m68k
163    COVERAGE_FORMAT=RTEMS
164    ;;
165  erc32|leon2|leon3)
166    CPU=sparc
167    COVERAGE_FORMAT=TSIM
168    ;;
169  edb7312|rtl22xx|gumstix|smdk2410)
170    CPU=arm
171    COVERAGE_FORMAT=Skyeye
172    RTEMS_EXTRA_ARGS="ON_SKYEYE=1"
173    ;;
174  *)
175    fatal "Unsupported BSP (${BSP}) for coverage analysis."
176    ;;
177esac
178
179if [ ${RTEMS_VERSION} = 4.6 ] ; then
180  TARGET=${CPU}-rtems
181else
182  TARGET=${CPU}-rtems${RTEMS_VERSION}
183fi
184
185##### VERBOSE
186print_verbose()
187{
188  echo "BSP:                " ${BSP}
189  echo "Target:             " ${TARGET}
190  echo "do_all:             " ${do_all}
191  echo "do_covmerge         " ${do_covmerge}
192  echo "do_posix:           " ${do_posix}
193  echo "do_developmental:   " ${do_developmental}
194  echo "do_optimize_size:   " ${do_optimize_size}
195  echo "do_rtems_update:    " ${do_rtems_update}
196  echo "do_rtems_configure: " ${do_rtems_configure}
197  echo "do_rtems_build:     " ${do_rtems_build}
198  echo "do_copy_tests:      " ${do_copy_tests}
199  echo "do_run_tests:       " ${do_run_tests}
200  echo "do_report:          " ${do_report}
201  echo "Clean Before:       " ${do_clean}
202}
203
204if [ ${verbose} = "yes" ] ; then
205  print_verbose
206fi
207
208# Basic error checking and sanity checking on the directory structure
209# and PATH
210if [ ! -d ${COVBASE} ] ; then
211  echo You do not have ${COVBASE}
212  exit 1
213fi
214
215if [ ! -d ${RTEMSDIR} ] ; then
216  echo "Check out RTEMS!!!"
217  exit 1
218fi
219
220grep _COVERAGE ${RTEMSDIR}/c/src/make/leaf.cfg >/dev/null
221if [ $? -ne 0 ] ; then
222  echo "It doesn't look like you patched the RTEMS source for coverage."
223  exit 1
224fi
225
226type ${TARGET}-gcc
227check_status $? "Path appears to be broken"
228
229# Start real action
230NM=${TARGET}-nm
231type ${NM}
232check_status $? "Cannot find ${NM} on PATH"
233
234if [ ${do_optimize_size} = yes ] ; then
235 c_opt=-Os
236else
237 c_opt=-O2
238fi
239
240EXTENSION=${c_opt}
241if [ ${do_posix} = yes ] ; then
242  EXTENSION=${EXTENSION}P
243else
244  EXTENSION=${EXTENSION}p
245fi
246
247if [ ${do_developmental} = yes ] ; then
248  EXTENSION=${EXTENSION}D
249else
250  EXTENSION=${EXTENSION}d
251fi
252
253day=`date +%Y%m%d`
254hour=`date +%H`
255minute=`date +%M`
256results_dir=${BSP}${EXTENSION}-${day}-${hour}${minute}
257
258echo "Testing ${TARGET}/${BSP} at optimization (${c_opt})"
259
260# print a data element in table
261print_element()
262{
263  echo "  <TD>"${*}"</TD>"
264}
265
266# generate table row of results in html
267generate_html()
268{
269  echo "<TR>"
270    print_element ${day}
271    print_element ${hour}:${minute}
272    print_element ${c_opt}
273    print_element ${do_posix}
274    print_element ${do_developmental}
275    print_element `grep "uncovered range" summary.txt | cut -d ' ' -f1`
276    print_element `grep "age Not Exec" summary.txt | cut -d ':' -f2`
277    print_element `grep "age Exec" summary.txt | cut -d ':' -f2`
278    print_element `grep "Bytes Not Exec" summary.txt | cut -d ':' -f2`
279    print_element `grep "Bytes Analyzed" summary.txt | cut -d ':' -f2`
280    print_element "<A HREF=\"${results_dir}\">untarred</A> " \
281                  "<A HREF=\"${results_dir}.tar.bz2\">tarball</A> "
282  echo "</TR>"
283}
284
285# generate the coverage report set
286generate_report()
287{
288  if [ $# -ne 3 ] ; then
289    echo Usage: ${0} TARGET BSP COVERAGE_FORMAT
290    exit 1
291  fi
292
293  TARGET=${1}
294  BSP=${2}
295  COVERAGE_FORMAT=${3}
296
297  cd ${BASEDIR}/${BSP}-tests/
298  check_status $? "cd ${BSP}-tests"
299
300  ${NM} -g -n hello.exe >hello.num
301
302  # Basic validity checking
303  tlow=`grep "T ${lowsym}$" hello.num | cut -d' ' -f1`
304  if [ X${tlow} = X ] ; then
305    echo "low symbol (${lowsym}) not found"
306  fi
307  thigh=`grep "T ${highsym}$" hello.num | cut -d' ' -f1`
308  if [ X${thigh} = X ] ; then
309    echo "high symbol (${highsym}) not found"
310  fi
311
312  if [ X${tlow} = X -o X${thigh} = X ] ; then
313    echo "Did you patch RTEMS for coverage testing?"
314    exit 1
315  fi
316
317  rm -f merged.cov report.txt
318
319  ( check_endof ; \
320    echo "Optimization Level : " ${c_opt} ; \
321    echo "POSIX Enabled      : " ${do_posix} ; \
322    echo "Developmental Code : " ${do_developmental} ; \
323  ) >summary.txt
324  ${COVBASE}/covmerge \
325    -l 0x${tlow} -h 0x${thigh} \
326    -r report.txt -s sizes.txt \
327    -E ${COVBASE}/Explanations.txt \
328    -f ${COVERAGE_FORMAT} \
329    -T ${TARGET} -e hello.exe -m merged.cov *.cov        >>summary.txt
330  check_status $? "covmerge failed"
331
332  (echo "====== Largest Range Sizes (Size and Count)======" ; \
333    grep ^Size report.txt | cut -d':' -f2 | sort -n | uniq -c | \
334       tail -15 | sed -e 's/ *\([0-9]*\) *\([0-9]*\)/\2 \1/' | sort -n -r | \
335       while read l; do printf "%6d %6d\n" $l; done ; \
336    echo "=====================================" ) >>summary.txt
337
338  generate_html >row.html
339
340  cat summary.txt
341
342  # Now create the archive of information
343  rm -rf ${results_dir}
344  mkdir ${results_dir}
345  cp summary.txt report.txt sizes.txt configuration.txt \
346     annotated.dmp hello.num row.html ${results_dir}
347  cp ${COVBASE}/Explanations.txt.NotFound ${results_dir}
348  cp hello.exe ${results_dir}
349  bzip2 -9 ${results_dir}/hello.exe
350  echo "Results saved in ${results_dir}.tar.bz2"
351  tar cjf ${results_dir}.tar.bz2 ${results_dir}
352  rm -rf ${results_dir}
353}
354
355# Now we are ready to start doing real work
356start=`date`
357
358# If necessary, clean up the RTEMS build and test run directories
359if [ ${do_clean} = "yes" ] ; then
360  echo "Cleaning before building"
361  rm -rf ${BASEDIR}/b-${BSP}
362  rm -rf ${BASEDIR}/${BSP}-tests
363else
364  echo "Skipping clean before building"
365fi
366
367# If they don't exist, create the RTEMS build and test run directories
368test -d ${BASEDIR}/b-${BSP}     || mkdir ${BASEDIR}/b-${BSP}
369test -d ${BASEDIR}/${BSP}-tests || mkdir ${BASEDIR}/${BSP}-tests
370
371# If requested, update and build the coverage support tools
372if [ ${do_covmerge} = "yes" ] ; then
373  echo "Updating and building covmerge..."
374
375  cd ${COVBASE}
376  check_status $? "cd covmerge"
377
378  cvs up -Pd 2>&1 | grep -v ^cvs
379  make clean all
380  check_status $? "build covmerge"
381
382  make
383  check_status $? "make covmerge"
384else
385  echo "Skipping Updating and building covmerge..."
386fi
387
388# If requested, update the RTEMS tree
389if [ ${do_rtems_update} = "yes" ] ; then
390  echo "Updating RTEMS ..."
391  cd ${RTEMSDIR}
392  check_status $? "cd rtems"
393
394  cvs up -Pd 2>&1 | grep -v ^cvs
395  ./bootstrap -c
396  ./bootstrap
397else
398  echo "Skipping Updating RTEMS ..."
399fi
400
401# Now let's patch the make/custom file
402custom=${RTEMSDIR}/make/custom/${BSP}.cfg
403if [ ! -r ${custom} ] ; then
404  echo "Unable to read ${custom}"
405  exit 1
406fi
407
408grep "^CFLAGS_OPTIMIZE_V.*=.*-O[2s]" ${custom} >/dev/null
409if [ $? -ne 0 ] ; then
410  echo "Unable to file CFLAGS_OPTIMIZE_V in ${custom}"
411  exit 1
412fi
413
414sed -e "s/-O[2s]/${c_opt}/" <${custom} >/tmp/XXX
415mv /tmp/XXX ${custom}
416
417# If requested, configure RTEMS
418if [ ${do_rtems_configure} = "yes" ] ; then
419  echo "Configuring RTEMS..."
420  rm -rf ${BASEDIR}/b-${BSP}/
421  mkdir ${BASEDIR}/b-${BSP}/
422  cd ${BASEDIR}/b-${BSP}/
423  check_status $? "cd b-${BSP}"
424
425  ##################
426  ################## WARNING!!!!!!
427  ##################
428  ################## BE CAREFUL ABOUT THIS CONFIGURE COMMAND.  IT IS
429  ################## VERY SPECIFIC TO COVERAGE TESTING
430  ##################
431  if [ ${do_posix} = "yes" ] ; then
432    posix_en=en
433  else
434    posix_en=dis
435  fi
436
437  if [ ${do_developmental} = "yes" ] ; then
438    EXPERIMENTAL_ARG="experimental"
439  else
440    EXPERIMENTAL_ARG="yes"
441  fi
442
443  ${RTEMSDIR}/configure NDEBUG=1 \
444    RTEMS_DO_NOT_INLINE_THREAD_ENABLE_DISPATCH=1 \
445    RTEMS_DO_NOT_INLINE_CORE_MUTEX_SEIZE=1 \
446    RTEMS_DO_NOT_UNROLL_THREADQ_ENQUEUE_PRIORITY=1 \
447    ${RTEMS_EXTRA_ARGS} \
448    --target=${TARGET} --enable-rtemsbsp=${BSP} \
449    --enable-maintainer-mode \
450    --enable-coverage=${EXPERIMENTAL_ARG} \
451    --disable-itron --${posix_en}able-posix --enable-tests \
452    --disable-tcpip --disable-ada --disable-cxx \
453    --prefix=${BASEDIR}/coverage/install >c.log 2>&1
454  check_status $? "configuring RTEMS for ${BSP}"
455else
456  echo "Skipping Configuring RTEMS ..."
457fi
458
459# If requested, build RTEMS
460if [ ${do_rtems_build} = "yes" ] ; then
461  echo "Building RTEMS..."
462
463  cd ${BASEDIR}/b-${BSP}/
464  check_status $? "cd b-${BSP}"
465
466  make -j4 >b.log 2>&1
467  check_status $? "Building RTEMS for ${BSP}"
468else
469  echo "Skipping Building RTEMS ..."
470fi
471
472# If requested, copy the tests from the build tree to the run tree
473if [ ${do_copy_tests} = "yes" ] ; then
474  echo "Copying tests..."
475
476  # clean destination
477  rm -rf ${BASEDIR}/${BSP}-tests/*
478  check_status $? "clean test directory"
479
480  cd ${BASEDIR}/b-${BSP}/
481  check_status $? "cd b-${BSP}"
482
483  cp `find . -name *.exe` ../${BSP}-tests
484else
485  echo "Skipping copying tests..."
486fi
487
488# If requested, run the tests with coverage reporting enabled
489if [ ${do_run_tests} = "yes" ] ; then
490  echo "Running tests..."
491  cd ${BASEDIR}/${BSP}-tests/
492  check_status $? "cd ${BSP}-tests"
493
494  time ${RUNNER} -c *.exe
495else
496  echo "Skipping Running tests..."
497fi
498
499# If requested, generate the coverage report
500if [ ${do_report} = "yes" ] ; then
501  echo "Generating report..."
502
503  # Generate the configuration settings file
504  ( \
505    echo "#"
506    echo "#  Settings for this coverage test run"
507    echo "#"
508    echo
509    echo "Compiler version:   " `${TARGET}-gcc --version  | grep gcc`
510    echo "Optimization flag   " ${c_opt}
511    print_verbose
512  ) > ${BASEDIR}/${BSP}-tests/configuration.txt
513
514  cd ${BASEDIR}/${BSP}-tests/
515  check_status $? "cd ${BSP}-tests"
516
517  generate_report ${TARGET} ${BSP} ${COVERAGE_FORMAT}
518else
519  echo "Skipping Generating report..."
520fi
521
522# If requested, publish the coverage report
523if [ ${do_publish} = "yes" ] ; then
524  cp ${BASEDIR}/${BSP}-tests/${results_dir}.tar.bz2 ${FTPDIR}
525  check_status $? "cp ${BSP} results to ftp site failed"
526
527  ${COVBASE}/generate_coverage_html ${FTPDIR}
528else
529  echo "Skipping Publishing report..."
530fi
531stop=`date`
532
533echo "Started: " ${start}
534echo "Stopped: " ${stop}
535
536exit 0
537
Note: See TracBrowser for help on using the repository browser.