From c4c410802565a27c9a2037d3c22f76ae384163da Mon Sep 17 00:00:00 2001
From: Eric van der Vlist <vdv@dyomedea.com>
Date: Fri, 20 Apr 2012 20:39:00 +0200
Subject: [PATCH] Starting to write pipeline actions that interact with an
 Heritrix server

---
 .../actions/heritrix-archive-set.xpl          | 120 ++++++++++++++++++
 archiver/pipelines/config.xml                 |   9 +-
 archiver/pipelines/default-log4j.xml          |  37 ++++++
 archiver/pipelines/post-archive-request.xpl   |   2 +-
 archiver/pipelines/properties.xml             |  34 +++++
 5 files changed, 198 insertions(+), 4 deletions(-)
 create mode 100644 archiver/pipelines/actions/heritrix-archive-set.xpl
 create mode 100644 archiver/pipelines/default-log4j.xml
 create mode 100644 archiver/pipelines/properties.xml

diff --git a/archiver/pipelines/actions/heritrix-archive-set.xpl b/archiver/pipelines/actions/heritrix-archive-set.xpl
new file mode 100644
index 0000000..b3693e4
--- /dev/null
+++ b/archiver/pipelines/actions/heritrix-archive-set.xpl
@@ -0,0 +1,120 @@
+
+<!--
+    
+    Create a new archive through Heritrix
+    
+-->
+
+<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline" xmlns:oxf="http://www.orbeon.com/oxf/processors" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xforms="http://www.w3.org/2002/xforms"
+    xmlns:xxforms="http://orbeon.org/oxf/xml/xforms" xmlns:exist="http://exist.sourceforge.net/NS/exist" xmlns:saxon="http://saxon.sf.net/"
+    xmlns:pipeline="java:org.orbeon.oxf.processor.pipeline.PipelineFunctionLibrary">
+
+    <p:param name="data" type="input"/>
+
+    <p:processor name="oxf:unsafe-xslt">
+        <p:input name="data" href="#data"/>
+        <p:input name="config">
+            <config xsl:version="2.0">
+                <relpath>queue.xml</relpath>
+                <operation>write</operation>
+                <type>xquery</type>
+                <parameter name="directory" type="string">
+                    <xsl:value-of select="translate(/action/@uuid, '-', '/')"/>
+                    <xsl:text>/</xsl:text>
+                </parameter>
+                <parameter name="filename" type="string">
+                    <xsl:value-of select="saxon:string-to-hexBinary(/action/@url, 'utf-8')"/>
+                    <xsl:text>.xml</xsl:text>
+                </parameter>
+                <parameter name="uuid" type="string">
+                    <xsl:value-of select="/action/@uuid"/>
+                </parameter>
+                <parameter name="url" type="string">
+                    <xsl:value-of select="/action/@url"/>
+                </parameter>
+                <parameter name="priority-resource" type="string">
+                    <xsl:value-of select="/action/@priority + 2"/>
+                </parameter>
+                <parameter name="priority-package" type="string">
+                    <xsl:value-of select="/action/@priority + 1"/>
+                </parameter>
+            </config>
+        </p:input>
+        <p:output name="data" id="data-access-data"/>
+    </p:processor>
+
+    <!--    <p:processor name="oxf:pipeline">
+        <p:input name="config" href="/data-access.xpl"/>
+        <p:input name="data" href="#data-access-data"/>
+        <p:input name="param">
+            <xquery><![CDATA[
+declare namespace util = "http://exist-db.org/xquery/util";
+
+for $q in /queue return
+    update 
+        insert (<action priority=$(priority-resource) uuid="{util:uuid()}" type="archive-resource" url=$(url) directory=$(directory) filename=$(filename)/>,
+               <action priority=$(priority-package) uuid="{util:uuid()}" type="package-archive" directory=$(directory)/>)
+        into $q,
+        
+for $a in /queue/action where $a/@uuid = $(uuid) return
+    update
+        delete $a
+        
+                ]]></xquery>
+        </p:input>
+        <p:output name="data" id="response" debug="response"/>
+    </p:processor>
+
+    <p:processor name="oxf:null-serializer">
+        <p:input name="data" href="#response"/>
+    </p:processor>
+-->
+    <p:processor name="oxf:pipeline">
+        <p:input name="config" href="/data-access.xpl"/>
+        <p:input name="data" transform="oxf:xslt" href="#data-access-data">
+            <config xsl:version="2.0">
+                <relpath>
+                    <xsl:value-of select="/config/parameter[@name='directory']"/>
+                    <xsl:text>index.xml</xsl:text>
+                </relpath>
+                <operation>write</operation>
+                <type>document</type>
+            </config>
+        </p:input>
+        <p:input name="param" transform="oxf:xslt" href="#data">
+            <archive-set xsl:version="2.0">
+                <xsl:copy-of select="/action/@url|/action/@uuid"/>
+            </archive-set>
+        </p:input>
+        <p:output name="data" id="response2" debug="response2"/>
+    </p:processor>
+
+    <p:processor name="oxf:null-serializer">
+        <p:input name="data" href="#response2"/>
+    </p:processor>
+
+    <!-- Create a new Heritrix job-->
+    <p:processor name="oxf:xforms-submission">
+        <p:input name="submission" transform="oxf:xslt" href="oxf:/config.xml">
+            <xforms:submission xsl:version="2.0" method="urlencoded-post" replace="none" action="{/config/heritrix/rest-api}" xxforms:authentication-scheme="digest">
+<xforms:header combine="replace"> 
+    <xforms:name>Accept</xforms:name>
+    <xforms:value>application/xml</xforms:value> 
+  </xforms:header> 
+</xforms:submission>
+        </p:input>
+        <p:input name="request" transform="oxf:xslt" href="#data">
+            <instance xsl:version="2.0">
+                <action>create</action>
+                <createpath>
+                    <xsl:value-of select="/action/@uuid"/>
+                </createpath>
+            </instance>
+        </p:input>
+        <p:output name="response" id="heritrix1" debug="heritrix1"/>
+    </p:processor>
+    <p:processor name="oxf:null-serializer">
+        <p:input name="data" href="#heritrix1"/>
+    </p:processor>
+
+</p:config>
diff --git a/archiver/pipelines/config.xml b/archiver/pipelines/config.xml
index 1060633..6894fe3 100644
--- a/archiver/pipelines/config.xml
+++ b/archiver/pipelines/config.xml
@@ -1,10 +1,13 @@
 <?xml version="1.0" encoding="utf-8"?>
 <config>
 
-    <exist-root>http://admin@localhost:8080/orbeon/exist/rest/db/</exist-root> 
+    <exist-root>http://admin@localhost:8080/orbeon/exist/rest/db/</exist-root>
     <exist-db>owark/</exist-db>
-    
+
     <user-agent>Mozilla/5.0 (compatible; owark/0.3; http://owark.org/)</user-agent>
 
+    <heritrix>
+        <rest-api>https://admin:envierse@localhost:8443/engine</rest-api>
+    </heritrix>
 
-</config>
\ No newline at end of file
+</config>
diff --git a/archiver/pipelines/default-log4j.xml b/archiver/pipelines/default-log4j.xml
new file mode 100644
index 0000000..28dc26f
--- /dev/null
+++ b/archiver/pipelines/default-log4j.xml
@@ -0,0 +1,37 @@
+<!--
+    Copyright (C) 2004 Orbeon, Inc.
+
+    This program is free software; you can redistribute it and/or modify it under the terms of the
+    GNU Lesser General Public License as published by the Free Software Foundation; either version
+    2.1 of the License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+    without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU Lesser General Public License for more details.
+
+    The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
+-->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" xmlns="http://jakarta.apache.org/log4j/">
+
+    <!-- This is the standard log appender to the console (System.out) -->
+    <appender name="ConsoleAppender" class="org.apache.log4j.ConsoleAppender">
+        <param name="Target" value="System.err"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d{ISO8601} %-5p %c %x - %m%n"/>
+        </layout>
+        <filter class="org.apache.log4j.varia.LevelRangeFilter">
+            <param name="LevelMin" value="INFO" />
+        </filter>
+    </appender>
+<!-- XForms engine activity -->
+<category name="org.orbeon.oxf.xforms.processor.XFormsServer">
+    <priority value="debug"/>
+</category>
+
+    <!-- This is the root logger -->
+    <root>
+        <priority value="debug"/>
+        <appender-ref ref="ConsoleAppender"/>
+    </root>
+
+</log4j:configuration>
diff --git a/archiver/pipelines/post-archive-request.xpl b/archiver/pipelines/post-archive-request.xpl
index 41df24a..8ab966c 100644
--- a/archiver/pipelines/post-archive-request.xpl
+++ b/archiver/pipelines/post-archive-request.xpl
@@ -23,7 +23,7 @@ declare namespace util = "http://exist-db.org/xquery/util";
 
 for $q in /queue return
     update 
-        insert <action priority="0" uuid="{util:uuid()}" type="archive-set" url="http://dyomedea.com"/> 
+        insert <action priority="0" uuid="{util:uuid()}" type="heritrix-archive-set" url="http://dyomedea.com"/> 
         into $q
                 
                 ]]></xquery>
diff --git a/archiver/pipelines/properties.xml b/archiver/pipelines/properties.xml
new file mode 100644
index 0000000..3dc77fc
--- /dev/null
+++ b/archiver/pipelines/properties.xml
@@ -0,0 +1,34 @@
+<!--
+    Copyright (C) 2004 Orbeon, Inc.
+
+    This program is free software; you can redistribute it and/or modify it under the terms of the
+    GNU Lesser General Public License as published by the Free Software Foundation; either version
+    2.1 of the License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+    without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU Lesser General Public License for more details.
+
+    The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
+-->
+<properties xmlns:xs="http://www.w3.org/2001/XMLSchema"
+            xmlns:oxf="http://www.orbeon.com/oxf/processors">
+
+    <property as="xs:anyURI"  name="oxf.log4j-config"                                value="oxf:/default-log4j.xml"/>
+    <property as="xs:integer" name="oxf.cache.size"                                  value="200"/>
+
+
+    <property as="xs:anyURI"
+        name="oxf.http.ssl.keystore.uri"
+        value="file:/var/local/heritrix-3.1.0/heritrix.keystore"/>
+    <property as="xs:string"
+        name="oxf.http.ssl.keystore.password"
+        value="heritrix"/>
+    <property as="xs:string"
+        name="oxf.http.ssl.hostname-verifier"
+        value="allow-all"/>
+
+<!--<property as="xs:NMTOKENS" name="oxf.xforms.logging.debug"
+    value="document model submission control event action analysis server server-body html resolver utils 
+           submission-details submission-body"/>-->
+</properties>