From 6f64c7f8a97f959f8389960c466fe54245ea83b9 Mon Sep 17 00:00:00 2001
From: Eric van der Vlist <vdv@dyomedea.com>
Date: Thu, 26 Apr 2012 14:13:24 +0200
Subject: [PATCH] Handling payload content types

---
 .../org/owark/orbeon/FromWarcConverter.java   | 29 ++++++++++++++
 .../src/org/owark/warc/WarcRecordContent.java | 40 ++++++++++++++++++-
 .../test/org/owark/warc/WarcParserTest.java   | 15 +++++--
 3 files changed, 80 insertions(+), 4 deletions(-)

diff --git a/archiver/java/src/org/owark/orbeon/FromWarcConverter.java b/archiver/java/src/org/owark/orbeon/FromWarcConverter.java
index ef3b672..493d98b 100644
--- a/archiver/java/src/org/owark/orbeon/FromWarcConverter.java
+++ b/archiver/java/src/org/owark/orbeon/FromWarcConverter.java
@@ -19,12 +19,19 @@ import org.orbeon.oxf.pipeline.api.XMLReceiver;
 import org.orbeon.oxf.processor.ProcessorImpl;
 import org.orbeon.oxf.processor.ProcessorInputOutputInfo;
 import org.orbeon.oxf.processor.ProcessorOutput;
+import org.orbeon.oxf.processor.ProcessorUtils;
 import org.orbeon.oxf.processor.serializer.BinaryTextXMLReceiver;
 import org.orbeon.oxf.util.NetUtils;
 import org.orbeon.oxf.xml.ContentHandlerHelper;
+import org.orbeon.oxf.xml.XMLConstants;
+import org.orbeon.oxf.xml.XMLUtils;
 import org.owark.warc.*;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.AttributesImpl;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 /**
  * This processor converts a WARC archive into an XML representation
@@ -99,6 +106,28 @@ public class FromWarcConverter extends ProcessorImpl {
                             }
                             helper.endElement();
                         }
+                        if (! content.endOfContent()) {
+                            helper.startPrefixMapping("xsi", "http://www.w3.org/2001/XMLSchema-instance");
+                            helper.startPrefixMapping("xs", "http://www.w3.org/2001/XMLSchema");
+                            String contentType = content.getPayloadContentType();
+                            AttributesImpl attributes = new AttributesImpl();
+                            attributes.addAttribute("", "content-type", "content-type", "CDATA", contentType);
+                            if (contentType.startsWith("text/") || contentType.matches(".*application/[^;]*xml.*")) {
+                                attributes.addAttribute(XMLConstants.XSI_URI, "type", "xsi:type", "CDATA", "xs:string");
+                                String encoding = content.getPayloadEncoding();
+                                if (encoding == null) {
+                                    encoding = "utf-8";
+                                }
+                                helper.startElement(ProcessorUtils.DEFAULT_TEXT_DOCUMENT_ELEMENT, attributes);
+                                XMLUtils.readerToCharacters(new InputStreamReader(content, encoding), xmlReceiver);
+                                helper.endElement();
+                            } else {
+                                attributes.addAttribute(XMLConstants.XSI_URI, "type", "xsi:type", "CDATA", "xs:base64Binary");
+                                helper.startElement(ProcessorUtils.DEFAULT_BINARY_DOCUMENT_ELEMENT, attributes);
+                                XMLUtils.inputStreamToBase64Characters(new BufferedInputStream(content), xmlReceiver);
+                                helper.endElement();
+                            }
+                        }
                         record.skipToEnd();
                         helper.endElement();
                         helper.endElement();
diff --git a/archiver/java/src/org/owark/warc/WarcRecordContent.java b/archiver/java/src/org/owark/warc/WarcRecordContent.java
index eb60634..ec0b841 100644
--- a/archiver/java/src/org/owark/warc/WarcRecordContent.java
+++ b/archiver/java/src/org/owark/warc/WarcRecordContent.java
@@ -17,6 +17,8 @@ package org.owark.warc;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Iterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Created by IntelliJ IDEA.
@@ -30,6 +32,7 @@ public class WarcRecordContent extends InputStream implements Iterator<WarcField
     private WarcRecord warcRecord;
     private Exception e;
     private String line;
+    private String payloadContentType;
 
     public WarcRecordContent(WarcRecord warcRecord) {
         this.warcRecord = warcRecord;
@@ -61,6 +64,9 @@ public class WarcRecordContent extends InputStream implements Iterator<WarcField
             return null;
         }
         WarcField field = new WarcField(line);
+        if (field.getKey().equals("Content-Type")) {
+            this.payloadContentType = field.getValue();
+        }
         line = null;
         return field;
     }
@@ -94,7 +100,7 @@ public class WarcRecordContent extends InputStream implements Iterator<WarcField
         return isHTTP() && isRequest();
     }
 
-    public Object endOfContent() {
+    public boolean endOfContent() {
         return warcRecord.isLimitReached();
     }
 
@@ -106,6 +112,38 @@ public class WarcRecordContent extends InputStream implements Iterator<WarcField
         return warcRecord.getContentLength();
     }
 
+    public String getPayloadContentType() {
+        String contentType = getPayloadContentHeader();
+        if (contentType != null && contentType.contains(";")) {
+            contentType = contentType.substring(0, contentType.indexOf(";"));
+        }
+        return contentType;
+    }
+
+    public String getPayloadContentHeader() {
+        String contentType = warcRecord.getContentType();
+        if (contentType.equals("application/warc-fields") || contentType.equals("application/http; msgtype=request")) {
+            return null;
+        }
+        if (contentType.equals("application/http; msgtype=response")) {
+            contentType = this.payloadContentType;
+        }
+        return contentType;
+    }
+
+    public String getPayloadEncoding() {
+        String contentType = getPayloadContentHeader();
+        if (contentType == null) {
+            return contentType;
+        }
+        Pattern pattern = Pattern.compile(".*;\\s*charset\\s*=\\s*([^;]+).*");
+        Matcher matcher = pattern.matcher(contentType);
+        if (matcher.matches()) {
+            return matcher.group(1).toLowerCase();
+        }
+        return null;
+     }
+
 
     public class HttpStatusLine {
 
diff --git a/archiver/java/test/org/owark/warc/WarcParserTest.java b/archiver/java/test/org/owark/warc/WarcParserTest.java
index 3a052e8..c0f1628 100644
--- a/archiver/java/test/org/owark/warc/WarcParserTest.java
+++ b/archiver/java/test/org/owark/warc/WarcParserTest.java
@@ -34,7 +34,7 @@ public class WarcParserTest {
         WarcParser warcParser = new WarcParser(new FileInputStream(file));
         Assert.assertEquals(true, warcParser.hasNext());
 
-        // RECORD
+        // RECORD (warcinfo)
 
         WarcRecord record = warcParser.next();
         Assert.assertEquals("WARC/1.0", warcParser.getMagic());
@@ -97,9 +97,12 @@ public class WarcParserTest {
         Assert.assertEquals("http-header-user-agent", field.getKey());
         Assert.assertEquals("Mozilla/5.0 (compatible; heritrix/3.1.0 +http://owark.org)", field.getValue());
         Assert.assertEquals(false, content.hasNext());
+        Assert.assertNull(content.getPayloadContentType());
+        Assert.assertNull(content.getPayloadContentHeader());
+        Assert.assertNull(content.getPayloadEncoding());
         Assert.assertEquals(true, content.endOfContent());
 
-        // Next record
+        // Next record (DNS response)
 
         Assert.assertEquals(true, warcParser.hasNext());
         record = warcParser.next();
@@ -134,9 +137,12 @@ public class WarcParserTest {
         Assert.assertEquals("dyomedea.com.\t\t1800\tIN\tA\t95.142.167.137", line);
         line = reader.readLine();
         Assert.assertEquals(true, content.endOfContent());
+        Assert.assertEquals("text/dns", content.getPayloadContentType());
+        Assert.assertEquals("text/dns", content.getPayloadContentHeader());
+        Assert.assertNull(content.getPayloadEncoding());
         Assert.assertNull(line);
 
-        // Next record
+        // Next record (HTTP response)
 
         Assert.assertEquals(true, warcParser.hasNext());
         record = warcParser.next();
@@ -190,6 +196,9 @@ public class WarcParserTest {
         Assert.assertEquals("<html><head><title>Apache Tomcat/6.0.24 - Rapport d'erreur</title>", line.substring(0, line.indexOf("<style>")));
         line = reader.readLine();
         Assert.assertNull(line);
+        Assert.assertEquals("text/html", content.getPayloadContentType());        
+        Assert.assertEquals("text/html;charset=utf-8", content.getPayloadContentHeader());
+        Assert.assertEquals("utf-8", content.getPayloadEncoding());
         Assert.assertEquals(true, content.endOfContent());