From ad356726034ec5ec7a2406f71de96f0ba7b75699 Mon Sep 17 00:00:00 2001
From: Eric van der Vlist <vdv@dyomedea.com>
Date: Sun, 15 Apr 2012 00:12:29 +0200
Subject: [PATCH] Still work in progress, but the WARC archive now validates
 with warc-tools' warcvalid.py...

---
 archiver/pipelines/actions/package-archive.xpl |  4 +++-
 archiver/pipelines/actions/warc-lib.xsl        | 12 +++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/archiver/pipelines/actions/package-archive.xpl b/archiver/pipelines/actions/package-archive.xpl
index 33cc1a6..da144d5 100644
--- a/archiver/pipelines/actions/package-archive.xpl
+++ b/archiver/pipelines/actions/package-archive.xpl
@@ -211,7 +211,9 @@ conformsTo:
             </xsl:variable>
             <document xsl:version="2.0" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string" content-type="text/plain">
               <xsl:apply-templates select="$request" mode="warc"/>
-              <xsl:apply-templates select="$response" mode="warc"/>
+              <xsl:apply-templates select="$response" mode="warc">
+                <xsl:with-param name="document-length" as="xs:integer" select="string-length(translate(/archive/response/document, ' &#xa;&#xd;', '')) * 3 div 4" tunnel="yes"/>
+              </xsl:apply-templates>
             </document>
           </xsl:template>
         </xsl:stylesheet>
diff --git a/archiver/pipelines/actions/warc-lib.xsl b/archiver/pipelines/actions/warc-lib.xsl
index a64c140..722100f 100644
--- a/archiver/pipelines/actions/warc-lib.xsl
+++ b/archiver/pipelines/actions/warc-lib.xsl
@@ -30,6 +30,9 @@
     </xsl:template>
 
     <xsl:template match="record" mode="warc">
+        <xsl:param name="document-length" as="xs:integer" select="0" tunnel="yes"/>
+        <xsl:call-template name="CRLF"/>
+        <xsl:call-template name="CRLF"/>
         <xsl:apply-templates select="header" mode="warc"/>
         <xsl:variable name="block">
             <xsl:apply-templates select="block" mode="warc"/>
@@ -38,15 +41,13 @@
             <field>
                 <name>Content-Length</name>
                 <value>
-                    <xsl:value-of select="string-length($block)"/>
+                    <xsl:value-of select="string-length($block) + $document-length "/>
                 </value>
             </field>
         </xsl:variable>
         <xsl:apply-templates select="$content-length" mode="warc"/>
         <xsl:call-template name="CRLF"/>
         <xsl:value-of select="$block"/>
-        <xsl:call-template name="CRLF"/>
-        <xsl:call-template name="CRLF"/>
     </xsl:template>
 
     <xsl:template match="block" mode="warc">
@@ -71,6 +72,11 @@
     </xsl:template>
 
     <xsl:template match="response" mode="warc-http">
+        <!--<xsl:message>
+            <xsl:value-of select="string-length(document)"/>
+            <xsl:text> - </xsl:text>
+            <xsl:value-of select="string-length(translate(document, ' &#xa;&#xd;', ''))"/>
+        </xsl:message>-->
         <line>
             <!-- TODO: get the HTTP version and status-->
             <xsl:text>HTTP/1.1 </xsl:text>