Improve execution of regression tests

- Make it possible to skip tests on some platforms
- Make it possible to expect a failure (typically: of the reference interpreter)
- Stop on error

4 files changed, 45 insertions, 16 deletions

diff --git a/test/regression/Makefile b/test/regression/Makefile
index d4ef2831..191a2285 100644
--- a/test/regression/Makefile
+++ b/test/regression/Makefile
@@ -22,18 +22,9 @@ TESTS=int32 int64 floats floats-basics \
 
 TESTS_COMP=attribs1 bitfields1 bitfields2 bitfields3 bitfields4 \
   bitfields5 bitfields6 bitfields7 bitfields8 \
-  builtins-$(ARCH) packedstruct2 alignas \
+  builtins-$(ARCH) packedstruct1 packedstruct2 alignas \
   varargs1 varargs2 sections alias
 
-# packedstruct1 makes unaligned memory accesses
-
-ifeq ($(ARCH),powerpc)
-TESTS_COMP+=packedstruct1
-endif
-ifeq ($(ARCH),x86)
-TESTS_COMP+=packedstruct1
-endif
-
 # Can run, both in compiled mode and in interpreter mode,
 # but produce processor-dependent results, so no reference output in Results
 
@@ -72,12 +63,12 @@ clean:
 
 test:
 	@echo "----------- Compiled tests -------------"
-	@for i in $(TESTS) $(TESTS_COMP); do \
+	@set -e; for i in $(TESTS) $(TESTS_COMP); do \
 	   SIMU='$(SIMU)' ./Runtest $$i ./$$i.compcert; \
          done
 	@echo "----------- Interpreted tests -------------"
-	@for i in $(TESTS); do \
-           SIMU='' ./Runtest $$i $(CCOMP) $(INTERPFLAGS) $$i.c; \
+	@set -e; for i in $(TESTS); do \
+           SIMU='' INTERP=1 ./Runtest $$i $(CCOMP) $(INTERPFLAGS) $$i.c; \
          done
 	@for i in $(TESTS_DIFF); do \
            if $(CCOMP) -fall -interp -quiet $$i.c > _cinterp.log; then \
diff --git a/test/regression/Runtest b/test/regression/Runtest
index ad2a58f1..f693219a 100755
--- a/test/regression/Runtest
+++ b/test/regression/Runtest
@@ -9,6 +9,9 @@ out="test$$.log"
 rm -f $out
 trap "rm -f $out" 0 INT QUIT
 
+# Is the test expected to fail?
+expect_fail=false
+
 # The architecture and the bitsize
 arch=`sed -n -e 's/^ARCH=//p' ../../Makefile.config`
 bits=`sed -n -e 's/^BITSIZE=//p' ../../Makefile.config`
@@ -26,10 +29,23 @@ else
   ref=""
 fi
 
+# Special conditions
+
+if test -f "$name.cond"; then
+  RUN=0 SKIP=1 EXPECT_FAIL=2 sh "$name.cond"
+  case "$?" in
+    1) echo "$name: skipped"; exit 0;;
+    2) expect_fail=true;;
+  esac
+fi
+
 # Administer the test
 if $SIMU $* > $out
 then
-  if test -n "$ref"; then
+  if $expect_fail; then
+    echo "$name: ERROR (should have failed but did not)"
+    exit 2
+  elif test -n "$ref"; then
     if cmp -s "$out" "$ref"; then
       echo "$name: passed"
       exit 0
@@ -43,7 +59,13 @@ then
     exit 0
   fi
 else
-  echo "$name: EXECUTION FAILED (status $?)"
-  exit 2
+  retcode=$?
+  if $expect_fail; then
+    echo "$name: passed (failed as expected)"
+    exit 0
+  else
+    echo "$name: EXECUTION FAILED (status $retcode)"
+    exit 2
+  fi
 fi
 
diff --git a/test/regression/funptr2.cond b/test/regression/funptr2.cond
new file mode 100644
index 00000000..ab460efb
--- /dev/null
+++ b/test/regression/funptr2.cond
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if test -n "$INTERP"
+then exit $EXPECT_FAIL
+else exit $RUN
+fi
diff --git a/test/regression/interop1.cond b/test/regression/interop1.cond
new file mode 100644
index 00000000..77904189
--- /dev/null
+++ b/test/regression/interop1.cond
@@ -0,0 +1,10 @@
+#!/bin/sh
+arch=`sed -n -e 's/^ARCH=//p' ../../Makefile.config`
+model=`sed -n -e 's/^MODEL=//p' ../../Makefile.config`
+system=`sed -n -e 's/^SYSTEM=//p' ../../Makefile.config`
+
+case "$arch,$model,$system" in
+  *,*,cygwin)  exit $SKIP;;
+  x86,32sse2,*|arm,*,*|powerpc,*,*)  exit $RUN;;
+  *) exit $SKIP;;
+esac