Merge pull request #744 from tls-attacker/performanceFix

Reworked the Transport module & XSD Validation

Author: ic0ns

Attacks/src/main/java/de/rub/nds/tlsattacker/attacks/task/InvalidCurveTask.java

@@ -59,6 +59,7 @@ public boolean execute() {
             }
 
             if (!state.getWorkflowTrace().executedAsPlanned()) {
+                LOGGER.debug("Not executed as planned!");
                 return false;
             }
             fingerprint = ResponseExtractor.getFingerprint(getState());

TLS-Core/src/main/java/de/rub/nds/tlsattacker/core/state/State.java

@@ -122,7 +122,7 @@ private WorkflowTrace loadWorkflowTrace() {
 
         if (config.getWorkflowInput() != null) {
             try {
-                trace = WorkflowTraceSerializer.read(new FileInputStream(new File(config.getWorkflowInput())));
+                trace = WorkflowTraceSerializer.secureRead(new FileInputStream(new File(config.getWorkflowInput())));
                 LOGGER.debug("Loaded workflow trace from " + config.getWorkflowInput());
             } catch (FileNotFoundException ex) {
                 LOGGER.warn("Could not read workflow trace. File not found: " + config.getWorkflowInput());

TLS-Core/src/main/java/de/rub/nds/tlsattacker/core/workflow/WorkflowTrace.java

@@ -68,7 +68,7 @@ public static WorkflowTrace copy(WorkflowTrace orig) {
         try {
             String origTraceStr = WorkflowTraceSerializer.write(orig);
             InputStream is = new ByteArrayInputStream(origTraceStr.getBytes(StandardCharsets.UTF_8.name()));
-            copy = WorkflowTraceSerializer.read(is);
+            copy = WorkflowTraceSerializer.insecureRead(is);
         } catch (JAXBException | IOException | XMLStreamException ex) {
             throw new ConfigurationException("Could not copy workflow trace: " + ex);
         }

TLS-Core/src/main/java/de/rub/nds/tlsattacker/core/workflow/WorkflowTraceSerializer.java

@@ -98,7 +98,7 @@ public static String write(WorkflowTrace trace) throws JAXBException, IOExceptio
 
     /**
      * @param  outputStream
-     *                       The OutputStream to which the Trace should be written to
+     *                       The OutputStream to which the Trace should be written to.
      * @param  workflowTrace
      *                       The WorkflowTrace that should be written
      * @throws JAXBException
@@ -135,7 +135,8 @@ public static void write(OutputStream outputStream, WorkflowTrace workflowTrace)
 
     /**
      * @param  inputStream
-     *                            The InputStream from which the Parameter should be read
+     *                            The InputStream from which the Parameter should be read. Does NOT perform schema
+     *                            validation
      * @return                    The deserialized WorkflowTrace
      * @throws JAXBException
      *                            JAXBException if the JAXB reports a problem
@@ -144,7 +145,71 @@ public static void write(OutputStream outputStream, WorkflowTrace workflowTrace)
      * @throws XMLStreamException
      *                            If there is a Problem with the XML Stream
      */
-    public static WorkflowTrace read(InputStream inputStream) throws JAXBException, IOException, XMLStreamException {
+    public static WorkflowTrace insecureRead(InputStream inputStream)
+        throws JAXBException, IOException, XMLStreamException {
+        context = getJAXBContext();
+        Unmarshaller unmarshaller = context.createUnmarshaller();
+        unmarshaller.setEventHandler(new ValidationEventHandler() {
+            @Override
+            public boolean handleEvent(ValidationEvent event) {
+                // raise an Exception also on Warnings
+                return false;
+            }
+        });
+        XMLInputFactory xif = XMLInputFactory.newFactory();
+        xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+        xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+        XMLStreamReader xsr = xif.createXMLStreamReader(inputStream);
+        WorkflowTrace wt = (WorkflowTrace) unmarshaller.unmarshal(xsr);
+        inputStream.close();
+        return wt;
+    }
+
+    /**
+     * Reads a file and does not perform schema validation
+     *
+     * @param  f
+     * @return
+     */
+    public static List<WorkflowTrace> insecureReadFolder(File f) {
+        if (f.isDirectory()) {
+            ArrayList<WorkflowTrace> list = new ArrayList<>();
+            for (File file : f.listFiles()) {
+                if (file.getName().startsWith(".")) {
+                    // We ignore the .gitignore File
+                    continue;
+                }
+                WorkflowTrace trace;
+                try {
+                    trace = WorkflowTraceSerializer.insecureRead(new FileInputStream(file));
+                    trace.setName(file.getAbsolutePath());
+                    list.add(trace);
+                } catch (JAXBException | IOException | XMLStreamException ex) {
+                    LOGGER.warn("Could not read " + file.getAbsolutePath() + " from Folder.");
+                    LOGGER.debug(ex.getLocalizedMessage(), ex);
+                }
+            }
+            return list;
+        } else {
+            throw new IllegalArgumentException("Cannot read Folder, because its not a Folder");
+        }
+
+    }
+
+    /**
+     * @param  inputStream
+     *                            The InputStream from which the Parameter should be read. Does perform schema
+     *                            validation
+     * @return                    The deserialized WorkflowTrace
+     * @throws JAXBException
+     *                            JAXBException if the JAXB reports a problem
+     * @throws IOException
+     *                            If something goes wrong while writing to the stream
+     * @throws XMLStreamException
+     *                            If there is a Problem with the XML Stream
+     */
+    public static WorkflowTrace secureRead(InputStream inputStream)
+        throws JAXBException, IOException, XMLStreamException {
         try {
             context = getJAXBContext();
             Unmarshaller unmarshaller = context.createUnmarshaller();
@@ -175,7 +240,13 @@ public boolean handleEvent(ValidationEvent event) {
         }
     }
 
-    public static List<WorkflowTrace> readFolder(File f) {
+    /**
+     * Reads a folder. Does perform schema validation.
+     *
+     * @param  f
+     * @return
+     */
+    public static List<WorkflowTrace> secureReadFolder(File f) {
         if (f.isDirectory()) {
             ArrayList<WorkflowTrace> list = new ArrayList<>();
             for (File file : f.listFiles()) {
@@ -185,7 +256,7 @@ public static List<WorkflowTrace> readFolder(File f) {
                 }
                 WorkflowTrace trace;
                 try {
-                    trace = WorkflowTraceSerializer.read(new FileInputStream(file));
+                    trace = WorkflowTraceSerializer.secureRead(new FileInputStream(file));
                     trace.setName(file.getAbsolutePath());
                     list.add(trace);
                 } catch (JAXBException | IOException | XMLStreamException ex) {

TLS-Core/src/main/java/de/rub/nds/tlsattacker/core/workflow/task/TlsTask.java

@@ -57,6 +57,7 @@ public ITask call() {
                     hasError = false;
                     break;
                 } else {
+                    LOGGER.debug("Could not execute task correctly. Increasing Timeout and reexecuting");
                     if (increasingSleepTimes) {
                         sleepTime += additionalSleepTime;
                     }

TLS-Core/src/main/resources/workflowTrace.xsd

@@ -2988,7 +2988,6 @@
 
   <xs:complexType name="transportHandler" abstract="true">
     <xs:sequence>
-      <xs:element name="isInStreamTerminating" type="xs:boolean"/>
       <xs:element name="timeout" type="xs:long"/>
     </xs:sequence>
   </xs:complexType>

TLS-Core/src/test/java/de/rub/nds/tlsattacker/core/protocol/XmlSerialisationTest.java

@@ -40,7 +40,7 @@ public void testProtocolMessages() throws Exception {
             trace.addTlsAction(new ReceiveAction(message));
             File f = folder.newFile();
             WorkflowTraceSerializer.write(f, trace);
-            WorkflowTrace newWorkflowTrace = WorkflowTraceSerializer.read(new FileInputStream(f));
+            WorkflowTrace newWorkflowTrace = WorkflowTraceSerializer.secureRead(new FileInputStream(f));
             assertTrue(newWorkflowTrace.getTlsActions().size() == 2);
 
             assertTrue("Message failed: " + message.getClass().getName(),
@@ -67,7 +67,7 @@ public void testExtensionMessages() throws Exception {
             trace.addTlsAction(new ReceiveAction(message));
             File f = folder.newFile();
             WorkflowTraceSerializer.write(f, trace);
-            WorkflowTrace newWorkflowTrace = WorkflowTraceSerializer.read(new FileInputStream(f));
+            WorkflowTrace newWorkflowTrace = WorkflowTraceSerializer.secureRead(new FileInputStream(f));
             assertTrue(newWorkflowTrace.getTlsActions().size() == 2);
 
             assertTrue("Extension failed: " + extension.getClass().getName(),

TLS-Core/src/test/java/de/rub/nds/tlsattacker/core/unittest/helper/FakeTransportHandler.java

@@ -71,4 +71,8 @@ public void closeClientConnection() throws IOException {
             opened = false;
     }
 
+    @Override
+    public void setTimeout(long timeout) {
+    }
+
 }

TLS-Core/src/test/java/de/rub/nds/tlsattacker/core/workflow/SerializationFullTest.java

@@ -158,7 +158,7 @@ public void test() throws JAXBException, IOException {
         }
         LOGGER.info(builder.toString());
         try {
-            trace = WorkflowTraceSerializer.read(new FileInputStream(f));
+            trace = WorkflowTraceSerializer.secureRead(new FileInputStream(f));
         } catch (XMLStreamException ex) {
             fail();
         }

TLS-Core/src/test/java/de/rub/nds/tlsattacker/core/workflow/WorkflowTraceNormalizerTestBadInput.java

@@ -135,7 +135,7 @@ private void loadTestVector(File testVectorPath) {
 
         try {
             trace = WorkflowTraceSerializer
-                .read(new ByteArrayInputStream(traceInputXml.getBytes(StandardCharsets.UTF_8.name())));
+                .secureRead(new ByteArrayInputStream(traceInputXml.getBytes(StandardCharsets.UTF_8.name())));
         } catch (JAXBException | IOException | XMLStreamException | DataBindingException ex) {
             LOGGER.error("Could not load workflow trace from test file " + testVectorPath + ": " + ex);
         }