import lit.formats
import lit.TestRunner

# Custom format class for static analyzer tests
class AnalyzerTest(lit.formats.ShTest):

    def execute(self, test, litConfig):
        result = self.executeWithAnalyzeSubstitution(
            test, litConfig, '-analyzer-constraints=range')

        if result.code == lit.Test.FAIL:
            return result

        # If z3 backend available, add an additional run line for it
        if test.config.clang_staticanalyzer_z3 == '1':
            result = self.executeWithAnalyzeSubstitution(
                test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3')

        return result

    def executeWithAnalyzeSubstitution(self, test, litConfig, substitution):
        saved_substitutions = list(test.config.substitutions)
        test.config.substitutions.append(('%analyze', substitution))
        result = lit.TestRunner.executeShTest(test, litConfig,
                                              self.execute_external)
        test.config.substitutions = saved_substitutions

        return result

import lit.formats
import lit.TestRunner

# Custom format class for static analyzer tests
class AnalyzerTest(lit.formats.ShTest):

    def execute(self, test, litConfig):
        results = []

        # Parse any test requirements ('REQUIRES: ')
        saved_test = test
        lit.TestRunner.parseIntegratedTestScript(test)

        if 'z3' not in test.requires:
            results.append(self.executeWithAnalyzeSubstitution(
                saved_test, litConfig, '-analyzer-constraints=range'))

            if results[-1].code == lit.Test.FAIL:
                return results[-1]

        # If z3 backend available, add an additional run line for it
        if test.config.clang_staticanalyzer_z3 == '1':
            results.append(self.executeWithAnalyzeSubstitution(
                saved_test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3'))

        # Combine all result outputs into the last element
        for x in results:
            if x != results[-1]:
                results[-1].output = x.output + results[-1].output

        if results:
            return results[-1]
        return lit.Test.Result(lit.Test.UNSUPPORTED,
            "Test requires the following unavailable features: z3")

    def executeWithAnalyzeSubstitution(self, test, litConfig, substitution):
        saved_substitutions = list(test.config.substitutions)
        test.config.substitutions.append(('%analyze', substitution))
        result = lit.TestRunner.executeShTest(test, litConfig,
            self.execute_external)
        test.config.substitutions = saved_substitutions

        return result

Improve test handling with multiple constraint managers
[analyzer]: Improve test handling with multiple constraint managers

Summary: Modify the test infrastructure to properly handle tests that require z3, and merge together the output of all tests on success. This is required for D28954.

Reviewers: dcoughlin, zaks.anna, NoQ, xazax.hun

Subscribers: cfe-commits

Differential Revision: https://reviews.llvm.org/D33308

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@305480 91177308-0d34-0410-b5e6-96231b3b80d8

  import lit.formats
  import lit.TestRunner
  
  # Custom format class for static analyzer tests
  class AnalyzerTest(lit.formats.ShTest):
  
      def execute(self, test, litConfig):
+         results = []
-         result = self.executeWithAnalyzeSubstitution(
-             test, litConfig, '-analyzer-constraints=range')
  
+         # Parse any test requirements ('REQUIRES: ')
+         saved_test = test
+         lit.TestRunner.parseIntegratedTestScript(test)
+ 
+         if 'z3' not in test.requires:
+             results.append(self.executeWithAnalyzeSubstitution(
+                 saved_test, litConfig, '-analyzer-constraints=range'))
+ 
-         if result.code == lit.Test.FAIL:
+             if results[-1].code == lit.Test.FAIL:
-             return result
+                 return results[-1]
  
          # If z3 backend available, add an additional run line for it
          if test.config.clang_staticanalyzer_z3 == '1':
-             result = self.executeWithAnalyzeSubstitution(
+             results.append(self.executeWithAnalyzeSubstitution(
-                 test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3')
+                 saved_test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3'))
  
+         # Combine all result outputs into the last element
+         for x in results:
+             if x != results[-1]:
+                 results[-1].output = x.output + results[-1].output
+ 
+         if results:
-         return result
+             return results[-1]
+         return lit.Test.Result(lit.Test.UNSUPPORTED,
+             "Test requires the following unavailable features: z3")
  
      def executeWithAnalyzeSubstitution(self, test, litConfig, substitution):
          saved_substitutions = list(test.config.substitutions)
          test.config.substitutions.append(('%analyze', substitution))
          result = lit.TestRunner.executeShTest(test, litConfig,
-                                               self.execute_external)
+             self.execute_external)
          test.config.substitutions = saved_substitutions
  
          return result
  
Improve test handling with multiple constraint managers
## Code Before:
import lit.formats
import lit.TestRunner

# Custom format class for static analyzer tests
class AnalyzerTest(lit.formats.ShTest):

    def execute(self, test, litConfig):
        result = self.executeWithAnalyzeSubstitution(
            test, litConfig, '-analyzer-constraints=range')

        if result.code == lit.Test.FAIL:
            return result

        # If z3 backend available, add an additional run line for it
        if test.config.clang_staticanalyzer_z3 == '1':
            result = self.executeWithAnalyzeSubstitution(
                test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3')

        return result

    def executeWithAnalyzeSubstitution(self, test, litConfig, substitution):
        saved_substitutions = list(test.config.substitutions)
        test.config.substitutions.append(('%analyze', substitution))
        result = lit.TestRunner.executeShTest(test, litConfig,
                                              self.execute_external)
        test.config.substitutions = saved_substitutions

        return result

## Instruction:
Improve test handling with multiple constraint managers
## Code After:
import lit.formats
import lit.TestRunner

# Custom format class for static analyzer tests
class AnalyzerTest(lit.formats.ShTest):

    def execute(self, test, litConfig):
        results = []

        # Parse any test requirements ('REQUIRES: ')
        saved_test = test
        lit.TestRunner.parseIntegratedTestScript(test)

        if 'z3' not in test.requires:
            results.append(self.executeWithAnalyzeSubstitution(
                saved_test, litConfig, '-analyzer-constraints=range'))

            if results[-1].code == lit.Test.FAIL:
                return results[-1]

        # If z3 backend available, add an additional run line for it
        if test.config.clang_staticanalyzer_z3 == '1':
            results.append(self.executeWithAnalyzeSubstitution(
                saved_test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3'))

        # Combine all result outputs into the last element
        for x in results:
            if x != results[-1]:
                results[-1].output = x.output + results[-1].output

        if results:
            return results[-1]
        return lit.Test.Result(lit.Test.UNSUPPORTED,
            "Test requires the following unavailable features: z3")

    def executeWithAnalyzeSubstitution(self, test, litConfig, substitution):
        saved_substitutions = list(test.config.substitutions)
        test.config.substitutions.append(('%analyze', substitution))
        result = lit.TestRunner.executeShTest(test, litConfig,
            self.execute_external)
        test.config.substitutions = saved_substitutions

        return result
