    001    /*
    002     * Copyright (C) 2000-2001 Iowa State University
    003     *
    004     * This file is part of mjc, the MultiJava Compiler.
    005     *
    006     * This program is free software; you can redistribute it and/or modify
    007     * it under the terms of the GNU General Public License as published by
    008     * the Free Software Foundation; either version 2 of the License, or
    009     * (at your option) any later version.
    010     *
    011     * This program is distributed in the hope that it will be useful,
    012     * but WITHOUT ANY WARRANTY; without even the implied warranty of
    013     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    014     * GNU General Public License for more details.
    015     *
    016     * You should have received a copy of the GNU General Public License
    017     * along with this program; if not, write to the Free Software
    018     * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
    019     *
    020     * $Id: TestFilesTestSuite.java,v 1.7 2005/01/15 03:21:43 cok Exp $
    021     * Author: David R. Cok
    022     */
    023    package junitutils;
    024    import junit.framework.*;
    025    import java.io.*;
    026    import java.util.Iterator;
    027    import java.lang.reflect.Method;
    028
    029    /** This is a JUnit TestSuite that is created from a number of tests as follows.
    030        Each TestCase is an instance of the inner class Helper, instantiated with
    031        a name of a file.  The file names are read from the file named by the
    032        parameter 'fileOfTestFilenames'.  The argument to the constructor named
    033        'args' provides a set of command-line arguments; the filename for the TestCase
    034        is added on to the end of the list of command-line arguments.  Then the static
    035        compile method of the given class is called on those command-line arguments.
    036    <P>
    037        The standard output and error output is captured from the execution of the
    038        compile method.  This is compared to the output in filename + "-expected".
    039        The TestCase succeeds if these match; if they do not match, the test fails and
    040        the actual output is saved in filename + "-ckd".
    041    <P>
    042        The test must be run from the directory in which it resides - because
    043        it creates and opens files in the current directory.
    044
    045        @author David R. Cok
    046    */
    047    public class TestFilesTestSuite  extends TestSuite {
    048
    049        //@ ghost public boolean initialized = false;
    050
    051        //@ ensures !initialized;
    052        protected TestFilesTestSuite() {}
    053
    054        /*
    055        Create derived classes with alternate tests by deriving a class from this
    056        one. It should contain an inner Helper class derived from
    057        TestFilesTestSuite.Helper.  The method TestFilesTestSuite.makeHelper
    058        should be overridden to return an instance of the derived Helper class.
    059        The derived Helper class should override Helper.dotest to do the actual
    060        test.
    061        */
    062
    063        // -------------------------------------------------------------
    064        // DATA MEMBERS
    065        // -------------------------------------------------------------
    066
    067        /** The name of this test suite. */
    068        //@ protected invariant initialized ==> (testName != null);
    069        protected String testName;
    070
    071        /** The method that is to be executed on the command-line arguments. */
    072        //@ protected invariant initialized ==> (method != null);
    073        protected Method method;
    074
    075        final static String SAVED_SUFFIX = "-ckd";
    076        final static String ORACLE_SUFFIX = "-expected";
    077
    078        // -------------------------------------------------------------
    079        // CONSTRUCTOR
    080        // -------------------------------------------------------------
    081
    082        /** A constructor for this test suite.
    083            @param testName The name of the test suite
    084            @param fileOfTestFilenames The file to be read for filenames of tests
    085            @param args     The command-line arguments that the static compile
    086                            method will be applied to, with the filename added on
    087            @param cls      The class in which to find the static compile method
    088        */
    089        //@ ensures initialized;
    090        public TestFilesTestSuite(/*@ non_null */ String testName,
    091                                    /*@ non_null */ String fileOfTestFilenames,
    092                                    String[] args, // Ignored! FIXME
    093                                    /*@ non_null */ Class cls
    094                                    ) {
    095            super(testName);
    096            this.testName = testName;
    097            try {
    098                method = cls.getMethod("compile", new Class[] { String[].class });
    099                //System.out.println("METHOD " + method);
    100            } catch (NoSuchMethodException e) {
    101                throw new RuntimeException(e.toString());
    102            }
    103
    104            try {
    105                Iterator i = new LineIterator(fileOfTestFilenames);
    106                while (i.hasNext()) {
    107                    String s = (String)i.next();
    108                    String[] allargs = Utils.parseLine(s);
    109                    s = allargs[allargs.length-1];
    110                    addTest(makeHelper(s,allargs));
    111                }
    112            } catch (java.io.IOException e) {
    113                throw new RuntimeException(e.toString());
    114            }
    115            //@ set initialized = true;
    116        }
    117
    118
    119        /** Factory method for the helper class object. */
    120        protected Helper makeHelper(String filename, String[] args) {
    121            return new Helper(filename,args);
    122        }
    123
    124
    125        // FIXME - This test does not do the equivalent of FIXTILT or PATHTOFILES
    126        // that is performed in the Makefile to canonicalize the outputs.  So far we
    127        // have not needed it.
    128
    129        /** This is a helper class that is actually a TestCase; it is run repeatedly
    130            with different constructor arguments.
    131        */
    132        public class Helper extends TestCase {
    133
    134            /**
    135                The first argument is used as the name of the test as well as
    136                the name of the file to be tested.
    137            */
    138            public Helper(String testname, String[] args) {
    139                super(testname);
    140                this.fileToTest = testname;
    141                this.args = args;
    142            }
    143
    144            /** Filename of comparison files */
    145            protected String fileToTest;
    146
    147            /** Result of test */
    148            protected Object returnedObject;
    149
    150            /** Command-line arguments (including filename) for this test. */
    151            protected String[] args;
    152
    153            /** This is the framework around the test.  It sets up the streams to
    154                capture output, and catches all relevant exceptions.
    155            */
    156            //@ also
    157            //@ requires initialized;
    158            public void runTest() throws java.io.IOException {
    159                //System.out.println("\nTest suite " + testName + ": "  + fileToTest);
    160                //for (int kk=0; kk<args.length; ++kk) System.out.println(args[kk]);
    161                //System.out.println();
    162
    163                PrintStream ps = null;
    164                ByteArrayOutputStream ba = new ByteArrayOutputStream(10000);
    165                try {
    166                    // Redirect the output to a file, and then do the compile
    167                    ps = new PrintStream(ba);
    168                    Utils.setStreams(ps);
    169
    170                    returnedObject = dotest(fileToTest,args);
    171
    172                } catch (IllegalAccessException e) {
    173                    Utils.restoreStreams(); // FIXME - One might think that the
    174                                            // argument to this and other calls
    175                                            // should be true - but for some reason
    176                                            // that makes the junittests take a long
    177                                            // time to run and to abort prematurely
    178                    fail(e.toString());
    179                } catch (IllegalArgumentException e) {
    180                    Utils.restoreStreams();
    181                    fail(e.toString());
    182                } catch (java.lang.reflect.InvocationTargetException e) {
    183                    Utils.restoreStreams();
    184                    java.io.StringWriter sw = new StringWriter();
    185                    sw.write(e.toString());
    186                    e.printStackTrace(new PrintWriter(sw));
    187                    fail(sw.toString());
    188                } catch (Throwable e) {  // THIS JUST FOR DEBUG
    189                    Utils.restoreStreams(); // Have to have this before the use of System.out on the next line
    190                    System.out.println(e);
    191                    e.printStackTrace();
    192                } finally {
    193                    Utils.restoreStreams();
    194                    if (ps != null) ps.close();
    195                }
    196                String err = doOutputCheck(fileToTest,ba.toString(),returnedObject);
    197                if (err != null) fail(err);
    198                //System.out.println("COMPLETED: " + fileToTest);
    199            }
    200        }
    201
    202        /** This is the actual test; it compiles the given file and compares its
    203            output to the expected result (in fileToTest+ORACLE_SUFFIX); the
    204            output is expected to
    205            match and the result of the compile to be true or false, depending on
    206            whether errors or warnings were reported.  Override this method in derived tests.
    207
    208        */
    209        //@ requires initialized;
    210        protected Object dotest(String fileToTest, String[] args)
    211                throws IllegalAccessException, IllegalArgumentException,
    212                                java.lang.reflect.InvocationTargetException {
    213
    214            return method.invoke(null,new Object[]{args});
    215        }
    216
    217
    218        //@ requires initialized;
    219        //@ requires fileToTest != null;
    220        //@ requires output != null;
    221        //@ requires returnedValue != null;
    222        protected String doOutputCheck(String fileToTest, String output,
    223                                    Object returnedValue) {
    224          try {
    225            String expectedOutput = Utils.readFile(fileToTest+ORACLE_SUFFIX);
    226            Diff df = new Diff("expected", expectedOutput, "actual", output);
    227
    228            if (!df.areDifferent()) {
    229                // If the two strings match, the test succeeds and we make sure
    230                // that there is no -ckd file to confuse anyone.
    231                (new File(fileToTest+SAVED_SUFFIX)).delete();
    232            } else {
    233                // If the strings do not match, we save the actual string and
    234                // fail the test.
    235                FileWriter f = null;
    236                try {
    237                    f = new FileWriter(fileToTest+SAVED_SUFFIX);
    238                    f.write(output);
    239                } finally {
    240                    if (f != null) f.close();
    241                }
    242
    243                return (df.result());
    244            }
    245            return checkReturnValue(fileToTest,expectedOutput,returnedValue);
    246          } catch (java.io.IOException e) {
    247            return (e.toString());
    248          }
    249        }
    250
    251        //@ requires initialized;
    252        //@ requires fileToTest != null;
    253        //@ requires expectedOutput != null;
    254        //@ requires returnedValue != null;
    255        public String checkReturnValue(String fileToTest, String expectedOutput,
    256                                            Object returnedValue) {
    257            if (returnedValue instanceof Boolean) {
    258                    return expectedStatusReport(fileToTest,
    259                                    ((Boolean)returnedValue).booleanValue(),
    260                                    expectedOutput);
    261            } else if (returnedValue instanceof Integer) {
    262                    return expectedStatusReport(fileToTest,
    263                                    ((Integer)returnedValue).intValue(),
    264                                    expectedOutput);
    265            } else {
    266                return ("The return value is of type " + returnedValue.getClass()
    267                    + " instead of int or boolean");
    268            }
    269        }
    270
    271
    272        /** Returns null if ok, otherwise returns failure message. */
    273        //@ requires initialized;
    274        //@ requires fileToTest != null;
    275        //@ requires expectedOutput != null;
    276        public String expectedStatusReport(String fileToTest,
    277                                    int ecode, String expectedOutput) {
    278            int ret = expectedIntegerStatus(fileToTest,expectedOutput);
    279            if (ecode == ret) return null;
    280            return "The compile produced an invalid return value.  It should be " + ret + " but instead is " + ecode;
    281        }
    282
    283        //@ requires initialized;
    284        //@ requires fileToTest != null;
    285        //@ requires expectedOutput != null;
    286        public String expectedStatusReport(String fileToTest,
    287                                    boolean b, String expectedOutput) {
    288            boolean status = expectedBooleanStatus(fileToTest,expectedOutput);
    289            if (status == b) return null;
    290            return ("The compile produced an invalid return value.  It should be "
    291                    + (!b) + " since there was " +
    292                    (b?"no ":"") + "error output but instead is " + b);
    293        }
    294
    295        //@ requires initialized;
    296        //@ requires fileToTest != null;
    297        //@ requires expectedOutput != null;
    298        public boolean expectedBooleanStatus(String fileToTest, String expectedOutput) {
    299            return expectedOutput.length()==0;
    300        }
    301
    302        //@ requires initialized;
    303        //@ requires fileToTest != null;
    304        //@ requires expectedOutput != null;
    305        public int expectedIntegerStatus(String fileToTest, String expectedOutput) {
    306            return 0;
    307        }
    308
    309    }
    310
    311