From 0e7bdd1de4d9764eafd2a0ce2612d65dbde4f6d0 Mon Sep 17 00:00:00 2001
From: Eric van der Vlist <vdv@dyomedea.com>
Date: Fri, 13 Apr 2012 18:01:53 +0200
Subject: [PATCH] Adding a basic squeleton to generate what should ultimately
 be a WARC archive

---
 archiver/pipelines/actions/archive-set.xpl    |   8 +-
 .../pipelines/actions/package-archive.xpl     | 292 ++++++++++++++++++
 archiver/pipelines/actions/warc-lib.xsl       |  68 ++++
 3 files changed, 366 insertions(+), 2 deletions(-)
 create mode 100644 archiver/pipelines/actions/package-archive.xpl
 create mode 100644 archiver/pipelines/actions/warc-lib.xsl

diff --git a/archiver/pipelines/actions/archive-set.xpl b/archiver/pipelines/actions/archive-set.xpl
index 9515c52..af4aa55 100644
--- a/archiver/pipelines/actions/archive-set.xpl
+++ b/archiver/pipelines/actions/archive-set.xpl
@@ -31,7 +31,10 @@
                 <parameter name="url" type="string">
                     <xsl:value-of select="/action/@url"/>
                 </parameter>
-                <parameter name="priority" type="string">
+                <parameter name="priority-resource" type="string">
+                    <xsl:value-of select="/action/@priority + 2"/>
+                </parameter>
+                <parameter name="priority-package" type="string">
                     <xsl:value-of select="/action/@priority + 1"/>
                 </parameter>
             </config>
@@ -48,7 +51,8 @@ declare namespace util = "http://exist-db.org/xquery/util";
 
 for $q in /queue return
     update 
-        insert <action priority=$(priority) uuid="{util:uuid()}" type="archive-resource" url=$(url) directory=$(directory) filename=$(filename)/> 
+        insert (<action priority=$(priority-resource) uuid="{util:uuid()}" type="archive-resource" url=$(url) directory=$(directory) filename=$(filename)/>,
+               <action priority=$(priority-package) uuid="{util:uuid()}" type="package-archive" directory=$(directory)/>)
         into $q,
         
 for $a in /queue/action where $a/@uuid = $(uuid) return
diff --git a/archiver/pipelines/actions/package-archive.xpl b/archiver/pipelines/actions/package-archive.xpl
new file mode 100644
index 0000000..83ea80d
--- /dev/null
+++ b/archiver/pipelines/actions/package-archive.xpl
@@ -0,0 +1,292 @@
+<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:saxon="http://saxon.sf.net/">
+
+  <p:param name="data" type="input"/>
+
+  <!-- Read the archive index -->
+  <p:processor name="oxf:pipeline">
+    <p:input name="config" href="/data-access.xpl"/>
+    <p:input name="data" transform="oxf:xslt" href="#data">
+      <config xsl:version="2.0">
+        <relpath>
+          <xsl:value-of select="/action/@directory"/>
+          <xsl:text>index.xml</xsl:text>
+        </relpath>
+        <operation>read</operation>
+        <type>document</type>
+      </config>
+    </p:input>
+    <p:input name="param">
+      <empty/>
+    </p:input>
+    <p:output name="data" id="index" debug="index"/>
+  </p:processor>
+
+  <!-- Create a WARC file -->
+  <p:processor name="oxf:file-serializer">
+    <p:input name="config">
+      <config>
+        <scope>request</scope>
+      </config>
+    </p:input>
+    <p:input name="data" transform="oxf:xslt" href="aggregate('root', #index, #data)">
+      <xsl:stylesheet version="2.0">
+        <xsl:import href="warc-lib.xsl"/>
+        <xsl:template match="/">
+          <xsl:variable name="content" as="node()*">
+            <version/>
+            <field>
+              <name>WARC-Type</name>
+              <value>warcinfo</value>
+            </field>
+            <field>
+              <name>WARC-Date</name>
+              <value>
+                <xsl:value-of select="current-dateTime()"/>
+              </value>
+            </field>
+            <field>
+              <name>WARC-Record-ID</name>
+              <value>
+                <xsl:text>&lt;urn:uuid:</xsl:text>
+                <xsl:value-of select="translate(substring(/root/action/@directory, 1, string-length(/root/action/@directory) - 1), '/', '-')"/>
+                <xsl:text>></xsl:text>
+              </value>
+            </field>
+            <field>
+              <name>Content-Type</name>
+              <value>application/warc-fields</value>
+            </field>
+            <!-- TODO: Content-Length: 381 -->
+            <CRLF/>
+            <field>
+              <name>software</name>
+              <value>Owark 0.3 http://owark.org</value>
+            </field>
+            <field>
+              <name>format</name>
+              <value>WARC file version 0.18</value>
+            </field>
+            <CRLF/>
+            <CRLF/>
+            <!--
+              
+              
+software: Heritrix 1.12.0 http://crawler.archive.org
+hostname: crawling017.archive.org
+ip: 207.241.227.234
+isPartOf: testcrawl-20050708
+description: testcrawl with WARC output
+operator: IA_Admin
+http-header-user-agent:
+ Mozilla/5.0 (compatible; heritrix/1.4.0 +http://crawler.archive.org)
+format: WARC file version 0.18
+conformsTo:
+ http://www.archive.org/documents/WarcFileFormat-0.18.html-->
+          </xsl:variable>
+          <document xsl:version="2.0" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string" content-type="text/plain">
+            <xsl:apply-templates select="$content" mode="warc"/>
+          </document>
+        </xsl:template>
+      </xsl:stylesheet>
+    </p:input>
+    <p:output name="data" id="warc" debug="warc"/>
+  </p:processor>
+
+  <!-- Loop over the index to retrieve the documents -->
+
+  <p:for-each href="#index" select="/archive-set/archive" id="files" root="files">
+
+    <!-- Read the document -->
+    <p:processor name="oxf:pipeline">
+      <p:input name="config" href="/data-access.xpl"/>
+      <p:input name="data" transform="oxf:xslt" href="aggregate('root', #data, current())">
+        <config xsl:version="2.0">
+          <relpath>
+            <xsl:value-of select="/root/action/@directory"/>
+            <xsl:value-of select="/root/archive/@href"/>
+          </relpath>
+          <operation>read</operation>
+          <type>document</type>
+        </config>
+      </p:input>
+      <p:input name="param">
+        <empty/>
+      </p:input>
+      <p:output name="data" id="document" debug="document"/>
+    </p:processor>
+
+    <!-- Add the request and start of response records -->
+    <p:processor name="oxf:file-serializer">
+      <p:input name="config" transform="oxf:xslt" href="#warc">
+        <config xsl:version="2.0">
+          <file>
+            <xsl:value-of select="substring-after(/url, 'file:')"/>
+          </file>
+          <make-directories>false</make-directories>
+          <append>true</append>
+        </config>
+      </p:input>
+      <p:input name="data" transform="oxf:xslt" href="#document">
+        <xsl:stylesheet version="2.0">
+          <xsl:import href="warc-lib.xsl"/>
+          <xsl:template match="/">
+            <xsl:variable name="content" as="node()*">
+              <!-- Request -->
+              <version/>
+              <field>
+                <name>WARC-Type</name>
+                <value>request</value>
+              </field>
+              <field>
+                <name>WARC-Target-URI</name>
+                <value>
+                  <xsl:value-of select="/archive/request/location"/>
+                </value>
+              </field>
+              <field>
+                <name>WARC-Date</name>
+                <value>
+                  <!-- TODO: replace that by the archive sate -->
+                  <xsl:value-of select="current-dateTime()"/>
+                </value>
+              </field>
+              <field>
+                <name>WARC-Record-ID</name>
+                <value>
+                  <xsl:text>&lt;urn:uuid:</xsl:text>
+                  <xsl:value-of select="translate(substring(/root/action/@directory, 1, string-length(/root/action/@directory) - 1), '/', '-')"/>
+                  <xsl:text>></xsl:text>
+                </value>
+              </field>
+              <field>
+                <name>Content-Type</name>
+                <value>application/http;msgtype=request</value>
+              </field>
+              <!-- TODO: Content-Length: 381 -->
+              <CRLF/>
+              <xsl:apply-templates select="/archive/request" mode="warc"/>
+              <CRLF/>
+              <CRLF/>
+              <!-- Response -->
+              <version/>
+              <field>
+                <name>WARC-Type</name>
+                <value>response</value>
+              </field>
+              <field>
+                <name>WARC-Target-URI</name>
+                <value>
+                  <xsl:value-of select="/archive/request/location"/>
+                </value>
+              </field>
+              <field>
+                <name>WARC-Date</name>
+                <value>
+                  <!-- TODO: replace that by the archive sate -->
+                  <xsl:value-of select="current-dateTime()"/>
+                </value>
+              </field>
+              <field>
+                <name>WARC-Record-ID</name>
+                <value>
+                  <xsl:text>&lt;urn:uuid:</xsl:text>
+                  <xsl:value-of select="translate(substring(/root/action/@directory, 1, string-length(/root/action/@directory) - 1), '/', '-')"/>
+                  <xsl:text>></xsl:text>
+                </value>
+              </field>
+              <field>
+                <name>Content-Type</name>
+                <value>application/http;msgtype=response</value>
+              </field>
+              <!-- TODO: Content-Length: 381 -->
+              <CRLF/>
+              <xsl:apply-templates select="/archive/response" mode="warc"/>
+              <CRLF/>
+              
+            </xsl:variable>
+            <document xsl:version="2.0" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string" content-type="text/plain">
+              <xsl:apply-templates select="$content" mode="warc"/>
+            </document>
+          </xsl:template>
+        </xsl:stylesheet>
+      </p:input>
+    </p:processor>
+
+    <!-- Add the response document to finalize the response record -->
+    <p:processor name="oxf:file-serializer">
+      <p:input name="config" transform="oxf:xslt" href="#warc">
+        <config xsl:version="2.0">
+          <file>
+            <xsl:value-of select="substring-after(/url, 'file:')"/>
+          </file>
+          <make-directories>false</make-directories>
+          <append>true</append>
+        </config>
+      </p:input>
+      <p:input name="data" href="#document#xpointer(/archive/response/document)"/>
+    </p:processor>
+    
+    <p:processor name="oxf:identity">
+      <p:input name="data" href="current()"/>
+      <p:output name="data" ref="files"/>
+    </p:processor>
+
+
+  </p:for-each>
+
+  <p:processor name="oxf:null-serializer">
+    <p:input name="data" href="#files"/>
+  </p:processor>
+
+  <p:processor name="oxf:zip">
+    <p:input name="data" transform="oxf:xslt" href="#warc">
+      <files xsl:version="2.0" file-name="archive.zip">
+        <file name="archive.warc">
+          <xsl:value-of select="/url"/>
+        </file>
+      </files>
+    </p:input>
+    <p:output name="data" id="zip"/>
+  </p:processor>
+
+  <p:processor name="oxf:file-serializer">
+    <p:input name="config">
+      <config>
+        <file>/tmp/archive.zip</file>
+      </config>
+    </p:input>
+    <p:input name="data" href="#zip"/>
+
+  </p:processor>
+
+  <!--
+  <!-\- Update the queue -\->
+  <p:processor name="oxf:pipeline">
+    <p:input name="config" href="/data-access.xpl"/>
+    <p:input name="data" transform="oxf:xslt" href="#data">
+      <config xsl:version="2.0">
+        <relpath>queue.xml</relpath>
+        <operation>write</operation>
+        <type>xquery</type>
+        <parameter name="uuid" type="string">
+          <xsl:value-of select="/action/@uuid"/>
+        </parameter>
+      </config>
+    </p:input>
+    <p:input name="param">
+      <xquery><![CDATA[
+                
+for $a in /queue/action where $a/@uuid = $(uuid) return
+    update
+        delete $a
+        
+                ]]></xquery>
+    </p:input>
+    <p:output name="data" id="response4" debug="response"/>
+  </p:processor>
+  <p:processor name="oxf:null-serializer">
+    <p:input name="data" href="#response4"/>
+  </p:processor>-->
+
+
+</p:config>
diff --git a/archiver/pipelines/actions/warc-lib.xsl b/archiver/pipelines/actions/warc-lib.xsl
new file mode 100644
index 0000000..c9b0a62
--- /dev/null
+++ b/archiver/pipelines/actions/warc-lib.xsl
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xd="http://www.oxygenxml.com/ns/doc/xsl" exclude-result-prefixes="xs xd"
+    version="2.0">
+    <xd:doc scope="stylesheet">
+        <xd:desc>
+            <xd:p><xd:b>Created on:</xd:b> Apr 13, 2012</xd:p>
+            <xd:p><xd:b>Author:</xd:b> vdv</xd:p>
+            <xd:p>Template library to produce WARC documents</xd:p>
+        </xd:desc>
+    </xd:doc>
+
+    <xsl:variable name="CRLF" select="'&#13;&#10;'"/>
+    <xsl:variable name="version">WARC/0.18</xsl:variable>
+    <xsl:template match="CRLF" mode="warc">
+        <xsl:value-of select="$CRLF"/>
+    </xsl:template>
+    <xsl:template match="version" mode="warc">
+        <xsl:value-of select="$version"/>
+        <xsl:value-of select="$CRLF"/>
+    </xsl:template>
+    <xsl:template match="field" mode="warc">
+        <xsl:value-of select="name"/>
+        <xsl:text>: </xsl:text>
+        <xsl:value-of select="value"/>
+        <xsl:value-of select="$CRLF"/>
+    </xsl:template>
+    <xsl:template match="line" mode="warc">
+        <xsl:value-of select="."/>
+        <xsl:value-of select="$CRLF"/>
+    </xsl:template>
+
+    <xsl:template match="request" mode="warc">
+        <line>
+            <xsl:value-of select="method"/>
+            <xsl:text> </xsl:text>
+            <xsl:value-of select="location"/>
+            <xsl:text> </xsl:text>
+            <!-- TODO: get the HTTP version -->
+            <xsl:text>HTTP/1.0</xsl:text>
+        </line>
+        <xsl:apply-templates select="header" mode="warc"/>
+    </xsl:template>
+
+    <xsl:template match="response" mode="warc">
+        <line>
+            <!-- TODO: get the HTTP version and status-->
+            <xsl:text>HTTP/1.1 </xsl:text>
+            <xsl:value-of select="code"/>
+            <xsl:text> OK</xsl:text>
+        </line>
+        <xsl:apply-templates select="header" mode="warc"/>
+    </xsl:template>
+
+    <xsl:template match="header" mode="warc">
+        <field>
+            <name>
+                <xsl:value-of select="@name"/>
+            </name>
+            <value>
+                <xsl:value-of select="."/>
+            </value>
+        </field>
+    </xsl:template>
+
+    <xsl:template match="text()" mode="warc"/>
+
+
+</xsl:stylesheet>