001    /*
002     * Copyright (C) 2000-2001 Iowa State University
003     *
004     * This file is part of mjc, the MultiJava Compiler.
005     *
006     * This program is free software; you can redistribute it and/or modify
007     * it under the terms of the GNU General Public License as published by
008     * the Free Software Foundation; either version 2 of the License, or
009     * (at your option) any later version.
010     *
011     * This program is distributed in the hope that it will be useful,
012     * but WITHOUT ANY WARRANTY; without even the implied warranty of
013     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
014     * GNU General Public License for more details.
015     *
016     * You should have received a copy of the GNU General Public License
017     * along with this program; if not, write to the Free Software
018     * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
019     *
020     * $Id: TestFilesTestSuite.java,v 1.7 2005/01/15 03:21:43 cok Exp $
021     * Author: David R. Cok
022     */
023    package junitutils;
024    import junit.framework.*;
025    import java.io.*;
026    import java.util.Iterator;
027    import java.lang.reflect.Method;
028    
029    /** This is a JUnit TestSuite that is created from a number of tests as follows.
030        Each TestCase is an instance of the inner class Helper, instantiated with
031        a name of a file.  The file names are read from the file named by the
032        parameter 'fileOfTestFilenames'.  The argument to the constructor named
033        'args' provides a set of command-line arguments; the filename for the TestCase
034        is added on to the end of the list of command-line arguments.  Then the static
035        compile method of the given class is called on those command-line arguments.
036    <P>
037        The standard output and error output is captured from the execution of the
038        compile method.  This is compared to the output in filename + "-expected".
039        The TestCase succeeds if these match; if they do not match, the test fails and
040        the actual output is saved in filename + "-ckd".
041    <P>
042        The test must be run from the directory in which it resides - because
043        it creates and opens files in the current directory.
044    
045        @author David R. Cok
046    */
047    public class TestFilesTestSuite  extends TestSuite {
048    
049        //@ ghost public boolean initialized = false;
050    
051        //@ ensures !initialized;
052        protected TestFilesTestSuite() {}
053    
054        /*
055        Create derived classes with alternate tests by deriving a class from this 
056        one. It should contain an inner Helper class derived from 
057        TestFilesTestSuite.Helper.  The method TestFilesTestSuite.makeHelper 
058        should be overridden to return an instance of the derived Helper class.  
059        The derived Helper class should override Helper.dotest to do the actual 
060        test.
061        */
062        
063        // -------------------------------------------------------------
064        // DATA MEMBERS
065        // -------------------------------------------------------------
066    
067        /** The name of this test suite. */
068        //@ protected invariant initialized ==> (testName != null);
069        protected String testName;
070    
071        /** The method that is to be executed on the command-line arguments. */
072        //@ protected invariant initialized ==> (method != null);
073        protected Method method;
074    
075        final static String SAVED_SUFFIX = "-ckd";
076        final static String ORACLE_SUFFIX = "-expected";
077    
078        // -------------------------------------------------------------
079        // CONSTRUCTOR
080        // -------------------------------------------------------------
081    
082        /** A constructor for this test suite.
083            @param testName The name of the test suite
084            @param fileOfTestFilenames The file to be read for filenames of tests
085            @param args     The command-line arguments that the static compile 
086                            method will be applied to, with the filename added on
087            @param cls      The class in which to find the static compile method
088        */
089        //@ ensures initialized;
090        public TestFilesTestSuite(/*@ non_null */ String testName, 
091                                    /*@ non_null */ String fileOfTestFilenames,
092                                    String[] args, // Ignored! FIXME
093                                    /*@ non_null */ Class cls
094                                    ) { 
095            super(testName);
096            this.testName = testName;
097            try {
098                method = cls.getMethod("compile", new Class[] { String[].class });
099                //System.out.println("METHOD " + method);
100            } catch (NoSuchMethodException e) {
101                throw new RuntimeException(e.toString());
102            }
103    
104            try { 
105                Iterator i = new LineIterator(fileOfTestFilenames);
106                while (i.hasNext()) {
107                    String s = (String)i.next();
108                    String[] allargs = Utils.parseLine(s);
109                    s = allargs[allargs.length-1];
110                    addTest(makeHelper(s,allargs));     
111                }
112            } catch (java.io.IOException e) {
113                throw new RuntimeException(e.toString());
114            }
115            //@ set initialized = true;
116        }
117    
118    
119        /** Factory method for the helper class object. */
120        protected Helper makeHelper(String filename, String[] args) {
121            return new Helper(filename,args);
122        }
123    
124    
125        // FIXME - This test does not do the equivalent of FIXTILT or PATHTOFILES
126        // that is performed in the Makefile to canonicalize the outputs.  So far we
127        // have not needed it.
128    
129        /** This is a helper class that is actually a TestCase; it is run repeatedly
130            with different constructor arguments.
131        */
132        public class Helper extends TestCase {
133        
134            /** 
135                The first argument is used as the name of the test as well as 
136                the name of the file to be tested.
137            */
138            public Helper(String testname, String[] args) {
139                super(testname);
140                this.fileToTest = testname;
141                this.args = args;
142            }
143            
144            /** Filename of comparison files */
145            protected String fileToTest;
146    
147            /** Result of test */
148            protected Object returnedObject;
149    
150            /** Command-line arguments (including filename) for this test. */
151            protected String[] args;
152        
153            /** This is the framework around the test.  It sets up the streams to
154                capture output, and catches all relevant exceptions.
155            */
156            //@ also
157            //@ requires initialized;
158            public void runTest() throws java.io.IOException {
159                //System.out.println("\nTest suite " + testName + ": "  + fileToTest);
160                //for (int kk=0; kk<args.length; ++kk) System.out.println(args[kk]);
161                //System.out.println();
162        
163                PrintStream ps = null;
164                ByteArrayOutputStream ba = new ByteArrayOutputStream(10000);
165                try {
166                    // Redirect the output to a file, and then do the compile
167                    ps = new PrintStream(ba);
168                    Utils.setStreams(ps);
169                    
170                    returnedObject = dotest(fileToTest,args);
171                    
172                } catch (IllegalAccessException e) {
173                    Utils.restoreStreams(); // FIXME - One might think that the
174                                            // argument to this and other calls
175                                            // should be true - but for some reason
176                                            // that makes the junittests take a long
177                                            // time to run and to abort prematurely
178                    fail(e.toString());
179                } catch (IllegalArgumentException e) {
180                    Utils.restoreStreams();
181                    fail(e.toString());
182                } catch (java.lang.reflect.InvocationTargetException e) {
183                    Utils.restoreStreams();
184                    java.io.StringWriter sw = new StringWriter();
185                    sw.write(e.toString());
186                    e.printStackTrace(new PrintWriter(sw));
187                    fail(sw.toString());
188                } catch (Throwable e) {  // THIS JUST FOR DEBUG
189                    Utils.restoreStreams(); // Have to have this before the use of System.out on the next line
190                    System.out.println(e);
191                    e.printStackTrace();
192                } finally {
193                    Utils.restoreStreams();
194                    if (ps != null) ps.close();
195                }
196                String err = doOutputCheck(fileToTest,ba.toString(),returnedObject);
197                if (err != null) fail(err);
198                //System.out.println("COMPLETED: " + fileToTest);
199            }
200        }
201         
202        /** This is the actual test; it compiles the given file and compares its 
203            output to the expected result (in fileToTest+ORACLE_SUFFIX); the 
204            output is expected to 
205            match and the result of the compile to be true or false, depending on
206            whether errors or warnings were reported.  Override this method in derived tests.
207             
208        */
209        //@ requires initialized;
210        protected Object dotest(String fileToTest, String[] args) 
211                throws IllegalAccessException, IllegalArgumentException, 
212                                java.lang.reflect.InvocationTargetException {
213                                                
214            return method.invoke(null,new Object[]{args});
215        }
216                
217        
218        //@ requires initialized;
219        //@ requires fileToTest != null;
220        //@ requires output != null;
221        //@ requires returnedValue != null;
222        protected String doOutputCheck(String fileToTest, String output, 
223                                    Object returnedValue) {
224          try {
225            String expectedOutput = Utils.readFile(fileToTest+ORACLE_SUFFIX);
226            Diff df = new Diff("expected", expectedOutput, "actual", output);
227    
228            if (!df.areDifferent()) {
229                // If the two strings match, the test succeeds and we make sure
230                // that there is no -ckd file to confuse anyone.
231                (new File(fileToTest+SAVED_SUFFIX)).delete();
232            } else {
233                // If the strings do not match, we save the actual string and
234                // fail the test.
235                FileWriter f = null;
236                try {
237                    f = new FileWriter(fileToTest+SAVED_SUFFIX);
238                    f.write(output);
239                } finally {
240                    if (f != null) f.close();
241                }
242                
243                return (df.result());
244            }
245            return checkReturnValue(fileToTest,expectedOutput,returnedValue);
246          } catch (java.io.IOException e) { 
247            return (e.toString()); 
248          }
249        }
250    
251        //@ requires initialized;
252        //@ requires fileToTest != null;
253        //@ requires expectedOutput != null;
254        //@ requires returnedValue != null;
255        public String checkReturnValue(String fileToTest, String expectedOutput,
256                                            Object returnedValue) {
257            if (returnedValue instanceof Boolean) {
258                    return expectedStatusReport(fileToTest,
259                                    ((Boolean)returnedValue).booleanValue(),
260                                    expectedOutput);
261            } else if (returnedValue instanceof Integer) {
262                    return expectedStatusReport(fileToTest,
263                                    ((Integer)returnedValue).intValue(),
264                                    expectedOutput);
265            } else {
266                return ("The return value is of type " + returnedValue.getClass()
267                    + " instead of int or boolean");
268            }
269        }
270    
271    
272        /** Returns null if ok, otherwise returns failure message. */
273        //@ requires initialized;
274        //@ requires fileToTest != null;
275        //@ requires expectedOutput != null;
276        public String expectedStatusReport(String fileToTest,
277                                    int ecode, String expectedOutput) {
278            int ret = expectedIntegerStatus(fileToTest,expectedOutput);
279            if (ecode == ret) return null;
280            return "The compile produced an invalid return value.  It should be " + ret + " but instead is " + ecode;
281        }
282    
283        //@ requires initialized;
284        //@ requires fileToTest != null;
285        //@ requires expectedOutput != null;
286        public String expectedStatusReport(String fileToTest,
287                                    boolean b, String expectedOutput) {
288            boolean status = expectedBooleanStatus(fileToTest,expectedOutput);
289            if (status == b) return null;
290            return ("The compile produced an invalid return value.  It should be "
291                    + (!b) + " since there was " +
292                    (b?"no ":"") + "error output but instead is " + b);
293        }
294    
295        //@ requires initialized;
296        //@ requires fileToTest != null;
297        //@ requires expectedOutput != null;
298        public boolean expectedBooleanStatus(String fileToTest, String expectedOutput) {
299            return expectedOutput.length()==0;
300        }
301    
302        //@ requires initialized;
303        //@ requires fileToTest != null;
304        //@ requires expectedOutput != null;
305        public int expectedIntegerStatus(String fileToTest, String expectedOutput) {
306            return 0;
307        }
308    
309    }
310    
311