Dyomedea
/
owark
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Cleaning the algorithm to compute friendly local names.

This commit is contained in:
Eric van der Vlist 2012-04-28 18:36:16 +02:00
parent cfaf8ae9c2
commit bf2980567a
3 changed files with 1763 additions and 1737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
archiver/pipelines/actions/resource-index.xslt
View File

@ -11,16 +11,41 @@
<xsl:variable name="source" select="/"/>
<xsl:variable name="common-extensions">
<content-type name="application/x-shockwave-flash">
<extension>swf</extension>
</content-type>
</xsl:variable>
<xsl:key name="extension" match="content-type" use="@name"/>
<xsl:function name="owk:add-extension" as="xs:string">
<xsl:param name="entry" as="element(entry)"/>
<xsl:param name="last-token" as="xs:string"/>
<xsl:sequence select="
if (contains($last-token, '.'))
then ''
else concat('.',
if (key('extension', $entry/content-type, $common-extensions))
then key('extension', $entry/content-type, $common-extensions)
else if (contains($entry/content-type, '/'))
then substring-after($entry/content-type, '/')
else 'unknown') "/>
</xsl:function>
<xsl:function name="owk:local-name" as="xs:string">
<xsl:param name="entry" as="element(entry)"/>
<xsl:variable name="is-seed" select="$entry/discovery-path='-'"/>
<xsl:variable name="tokens" select="tokenize($entry/uri, '/')"/>
<xsl:variable name="tokens" select="tokenize(if (contains($entry/uri, '?')) then substring-before($entry/uri, '?') else $entry/uri, '/')"/>
<xsl:sequence
select="if ($is-seed)
then 'index.html'
else concat($tokens[3], '/', if ($tokens[last()] = '')
then concat('index.', substring-after($entry/content-type, '/') )
else $tokens[last()])"
else concat(
$tokens[3],
'/',
if ($tokens[last()] = '') then 'index' else $tokens[last()],
owk:add-extension($entry, $tokens[last()]))"
/>
</xsl:function>
@ -32,11 +57,12 @@
then $local-name
else concat(
substring-before($local-name, '/'),
substring-before(concat(substring-after($local-name, '/'), '.'), '.'),
'/',
substring-before(substring-after($local-name, '/'), '.'),
'-',
count($entry/preceding-sibling::entry[owk:local-name(.) = $local-name]) + 1,
if (contains(substring-after($local-name, '/'), '.'))
then concat(substring-after(substring-after($local-name, '/'), '.'), '.')
else ''
'.',
substring-after(substring-after($local-name, '/'), '.')
)"
/>
</xsl:function>

84
archiver/pipelines/test/local-names.xml
View File

@ -1,4 +1,4 @@
<index>
<index>
<resource>
<uri seed="true">http://eric.van-der-vlist.com/blog/</uri>
<local-name>index.html</local-name>
@ -258,7 +258,7 @@
</resource>
<resource>
<uri seed="false">http://gmpg.org/xfn/11</uri>
<local-name>gmpg.org/11</local-name>
<local-name>gmpg.org/11-1.html</local-name>
<type>html</type>
<embeds>http://gmpg.org/xfn/faq</embeds>
<embeds>http://gmpg.org/xfn/intro</embeds>
@ -267,7 +267,7 @@
</resource>
<resource>
<uri seed="false">http://www.dailymotion.com/embed/video/xq0c5g</uri>
<local-name>www.dailymotion.comxq0c5g1</local-name>
<local-name>www.dailymotion.com/xq0c5g.html</local-name>
<type>html</type>
<embeds>http://static1.dmcdn.net/images/black-loading.gif</embeds>
<embeds>http://ajax.aspnetcdn.com/ajax/jQuery/jquery-1.6.4.js</embeds>
@ -276,13 +276,13 @@
</resource>
<resource>
<uri seed="false">http://gmpg.org/xfn/11.html</uri>
<local-name>gmpg.org/11.html</local-name>
<local-name>gmpg.org/11-2.html</local-name>
<type>html</type>
<same-as seed="false">http://gmpg.org/xfn/11</same-as>
</resource>
<resource>
<uri seed="false">http://www.dailymotion.com/sense/images/vs/tag.gif?routename=embed_player&amp;pagename=player%2Fexternal&amp;embedder=http%3A%2F%2Feric.van-der-vlist.com%2Fblog%2F&amp;video=43685764&amp;vchan=news&amp;streammode=none&amp;videotitle=%231+Clip+officiel+de+campagne+de+Fran%C3%A7ois+Hollande&amp;explicit=0&amp;videotype=official&amp;user=francoishollande&amp;usertype=official&amp;revshare=1&amp;locale=fr&amp;asnum=AS12322&amp;country=FR&amp;lat=48.8667000000000&amp;lng=2.3333000000000&amp;rid=1335599021&amp;urange=66&amp;ff=on&amp;log=1</uri>
<local-name>www.dailymotion.com/tag.gif?routename=embed_player&amp;pagename=player%2Fexternal&amp;embedder=http%3A%2F%2Feric.van-der-vlist.com%2Fblog%2F&amp;video=43685764&amp;vchan=news&amp;streammode=none&amp;videotitle=%231+Clip+officiel+de+campagne+de+Fran%C3%A7ois+Hollande&amp;explicit=0&amp;videotype=official&amp;user=francoishollande&amp;usertype=official&amp;revshare=1&amp;locale=fr&amp;asnum=AS12322&amp;country=FR&amp;lat=48.8667000000000&amp;lng=2.3333000000000&amp;rid=1335599021&amp;urange=66&amp;ff=on&amp;log=1</local-name>
<local-name>www.dailymotion.com/tag.gif</local-name>
<type>gif</type>
</resource>
<resource>
@ -297,17 +297,17 @@
</resource>
<resource>
<uri seed="false">http://gmpg.org/xfn/faq</uri>
<local-name>gmpg.org/faq</local-name>
<local-name>gmpg.org/faq-1.html</local-name>
<type>html</type>
</resource>
<resource>
<uri seed="false">http://www.dailymotion.com/swf/video/xq0c5g</uri>
<local-name>www.dailymotion.comxq0c5g2</local-name>
<local-name>www.dailymotion.com/xq0c5g.swf</local-name>
<type>x-shockwave-flash</type>
</resource>
<resource>
<uri seed="false">http://static2.dmcdn.net/static/video/467/586/43685764:jpeg_preview_large.jpg?20120419190724</uri>
<local-name>static2.dmcdn.net/43685764:jpeg_preview_large.jpg?20120419190724</local-name>
<local-name>static2.dmcdn.net/43685764:jpeg_preview_large.jpg</local-name>
<type>jpeg</type>
</resource>
<resource>
@ -322,13 +322,13 @@
</resource>
<resource>
<uri seed="false">http://gmpg.org/xfn/faq.html</uri>
<local-name>gmpg.org/faq.html</local-name>
<local-name>gmpg.org/faq-2.html</local-name>
<type>html</type>
<same-as seed="false">http://gmpg.org/xfn/faq</same-as>
</resource>
<resource>
<uri seed="false">http://www.dailymotion.com/logger/video/access/xq0c5g?session_id=&amp;referer=http%3A%2F%2Feric.van-der-vlist.com%2Fblog%2F&amp;country=FR&amp;lon=2.3333000000000&amp;lat=48.8667000000000&amp;blog_key=&amp;key=50l97s23f00n0adrsm36r20&amp;v=4f9bc978&amp;i=52ec2038&amp;h=59fb555d779dce1d464f4e80975f9d5e</uri>
<local-name>www.dailymotion.com/xq0c5g?session_id=&amp;referer=http%3A%2F%2Feric.van-der-vlist.com%2Fblog%2F&amp;country=FR&amp;lon=2.3333000000000&amp;lat=48.8667000000000&amp;blog_key=&amp;key=50l97s23f00n0adrsm36r20&amp;v=4f9bc978&amp;i=52ec2038&amp;h=59fb555d779dce1d464f4e80975f9d5e</local-name>
<local-name>www.dailymotion.com/xq0c5g.gif</local-name>
<type>gif</type>
</resource>
<resource>
@ -343,7 +343,7 @@
</resource>
<resource>
<uri seed="false">http://static2.dmcdn.net/static/video/467/586/43685764:jpeg_preview_source.jpg?20120419190724</uri>
<local-name>static2.dmcdn.net/43685764:jpeg_preview_source.jpg?20120419190724</local-name>
<local-name>static2.dmcdn.net/43685764:jpeg_preview_source.jpg</local-name>
<type>jpeg</type>
</resource>
<resource>
@ -368,12 +368,12 @@
</resource>
<resource>
<uri seed="false">http://www.dailymotion.com/video/xq0c5g_1-clip-officiel-de-campagne-de-francois-hollande_news</uri>
<local-name>www.dailymotion.com/xq0c5g_1-clip-officiel-de-campagne-de-francois-hollande_news</local-name>
<local-name>www.dailymotion.com/xq0c5g_1-clip-officiel-de-campagne-de-francois-hollande_news.html</local-name>
<type>html</type>
</resource>
<resource>
<uri seed="false">http://gmpg.org/xfn/intro</uri>
<local-name>gmpg.org/intro</local-name>
<local-name>gmpg.org/intro-1.html</local-name>
<type>html</type>
</resource>
<resource>
@ -393,13 +393,13 @@
</resource>
<resource>
<uri seed="false">http://gmpg.org/xfn/intro.html</uri>
<local-name>gmpg.org/intro.html</local-name>
<local-name>gmpg.org/intro-2.html</local-name>
<type>html</type>
<same-as seed="false">http://gmpg.org/xfn/intro</same-as>
</resource>
<resource>
<uri seed="false">http://static2.dmcdn.net/static/video/467/586/43685764:jpeg_preview_medium.jpg?20120419190724</uri>
<local-name>static2.dmcdn.net/43685764:jpeg_preview_medium.jpg?20120419190724</local-name>
<local-name>static2.dmcdn.net/43685764:jpeg_preview_medium.jpg</local-name>
<type>jpeg</type>
</resource>
<resource>
@ -439,7 +439,7 @@
</resource>
<resource>
<uri seed="false">http://www.dailymotion.com/behavior/%session%/xq0c5g</uri>
<local-name>www.dailymotion.comxq0c5g3</local-name>
<local-name>www.dailymotion.com/xq0c5g.unknown</local-name>
<type/>
</resource>
<resource>
@ -594,12 +594,12 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/shutter-reloaded/shutter-reloaded.js?ver=2.4</uri>
<local-name>eric.van-der-vlist.com/shutter-reloaded.js?ver=2.4</local-name>
<local-name>eric.van-der-vlist.com/shutter-reloaded.js</local-name>
<type>javascript</type>
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/comments/feed/</uri>
<local-name>eric.van-der-vlist.comindex1xml.</local-name>
<local-name>eric.van-der-vlist.com/index-1.xml</local-name>
<type>xml</type>
</resource>
<resource>
@ -619,7 +619,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/shutter-reloaded/shutter-reloaded.css?ver=2.4</uri>
<local-name>eric.van-der-vlist.com/shutter-reloaded.css?ver=2.4</local-name>
<local-name>eric.van-der-vlist.com/shutter-reloaded.css</local-name>
<type>css</type>
</resource>
<resource>
@ -664,7 +664,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/sociable-30/sociable.css?ver=5.13</uri>
<local-name>eric.van-der-vlist.com/sociable.css?ver=5.13</local-name>
<local-name>eric.van-der-vlist.com/sociable.css</local-name>
<type>css</type>
</resource>
<resource>
@ -784,7 +784,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/xmlrpc.php?rsd</uri>
<local-name>eric.van-der-vlist.com/xmlrpc.php?rsd</local-name>
<local-name>eric.van-der-vlist.com/xmlrpc-1.php</local-name>
<type>xml</type>
</resource>
<resource>
@ -824,7 +824,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/feed/</uri>
<local-name>eric.van-der-vlist.comindex2xml.</local-name>
<local-name>eric.van-der-vlist.com/index-2.xml</local-name>
<type>xml</type>
</resource>
<resource>
@ -859,7 +859,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/openid/f/openid.js?ver=519</uri>
<local-name>eric.van-der-vlist.com/openid.js?ver=519</local-name>
<local-name>eric.van-der-vlist.com/openid.js</local-name>
<type>javascript</type>
</resource>
<resource>
@ -879,7 +879,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-includes/js/jquery/jquery.js?ver=1.7.1</uri>
<local-name>eric.van-der-vlist.com/jquery.js?ver=1.7.1</local-name>
<local-name>eric.van-der-vlist.com/jquery.js</local-name>
<type>javascript</type>
</resource>
<resource>
@ -1114,7 +1114,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/nextgen-gallery/css/nggallery.css?ver=1.0.0</uri>
<local-name>eric.van-der-vlist.com/nggallery.css?ver=1.0.0</local-name>
<local-name>eric.van-der-vlist.com/nggallery.css</local-name>
<type>css</type>
<embeds>http://eric.van-der-vlist.com/blog/wp-content/plugins/nextgen-gallery/css/albumset.gif</embeds>
</resource>
@ -1240,7 +1240,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/nextgen-gallery/js/jquery.cycle.all.min.js?ver=2.9995</uri>
<local-name>eric.van-der-vlist.com/jquery.cycle.all.min.js?ver=2.9995</local-name>
<local-name>eric.van-der-vlist.com/jquery.cycle.all.min.js</local-name>
<type>javascript</type>
</resource>
<resource>
@ -1295,7 +1295,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/wp-note/style.css</uri>
<local-name>eric.van-der-vlist.comstyle1css.</local-name>
<local-name>eric.van-der-vlist.com/style-1.css</local-name>
<type>css</type>
<embeds>http://eric.van-der-vlist.com/blog/wp-content/plugins/wp-note/images/important.png</embeds>
<embeds>http://eric.van-der-vlist.com/blog/wp-content/plugins/wp-note/images/help.png</embeds>
@ -1330,7 +1330,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/xmlrpc.php</uri>
<local-name>eric.van-der-vlist.com/xmlrpc.php</local-name>
<local-name>eric.van-der-vlist.com/xmlrpc-2.php</local-name>
<type>text</type>
</resource>
<resource>
@ -1385,7 +1385,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/sociable-30/addtofavorites.js?ver=5.13</uri>
<local-name>eric.van-der-vlist.com/addtofavorites.js?ver=5.13</local-name>
<local-name>eric.van-der-vlist.com/addtofavorites.js</local-name>
<type>javascript</type>
</resource>
<resource>
@ -1435,7 +1435,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/themes/twentyten/style.css</uri>
<local-name>eric.van-der-vlist.comstyle2css.</local-name>
<local-name>eric.van-der-vlist.com/style-2.css</local-name>
<type>css</type>
<embeds>http://eric.van-der-vlist.com/blog/wp-content/themes/twentyten/images/wordpress.png</embeds>
</resource>
@ -1571,7 +1571,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/nextgen-gallery/js/ngg.slideshow.min.js?ver=1.06</uri>
<local-name>eric.van-der-vlist.com/ngg.slideshow.min.js?ver=1.06</local-name>
<local-name>eric.van-der-vlist.com/ngg.slideshow.min.js</local-name>
<type>javascript</type>
</resource>
<resource>
@ -1611,7 +1611,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/wp-content/plugins/openid/f/openid.css?ver=519</uri>
<local-name>eric.van-der-vlist.com/openid.css?ver=519</local-name>
<local-name>eric.van-der-vlist.com/openid.css</local-name>
<type>css</type>
<embeds>http://eric.van-der-vlist.com/blog/wp-content/plugins/openid/f/ajax-loader.gif</embeds>
<embeds>http://eric.van-der-vlist.com/blog/wp-content/plugins/openid/f/openid.gif</embeds>
@ -1633,40 +1633,40 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/</uri>
<local-name>eric.van-der-vlist.comindex2html.</local-name>
<local-name>eric.van-der-vlist.com/index-2.html</local-name>
<type>html</type>
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/2008/01/10/to-xforms-or-not-to-xforms/</uri>
<local-name>eric.van-der-vlist.comindex4html.</local-name>
<local-name>eric.van-der-vlist.com/index-4.html</local-name>
<type>html</type>
<same-as seed="false">http://eric.van-der-vlist.com/blog/L/t</same-as>
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/?a.offsetWidth:a.offsetHeight,e=b===</uri>
<local-name>eric.van-der-vlist.com/?a.offsetWidth:a.offsetHeight,e=b===</local-name>
<local-name>eric.van-der-vlist.com/index-5.html</local-name>
<type>html</type>
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/?this.style.display=</uri>
<local-name>eric.van-der-vlist.com/?this.style.display=</local-name>
<local-name>eric.van-der-vlist.com/index-6.html</local-name>
<type>html</type>
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/2005/11/22/1614_the_complex_simple_problem_of_media_types/</uri>
<local-name>eric.van-der-vlist.comindex6html.</local-name>
<local-name>eric.van-der-vlist.com/index-8.html</local-name>
<type>html</type>
<same-as seed="false">http://eric.van-der-vlist.com/blog/.16</same-as>
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/2005/12/23/1737_treebind_data_binding_and_design_patterns/</uri>
<local-name>eric.van-der-vlist.comindex7html.</local-name>
<local-name>eric.van-der-vlist.com/index-9.html</local-name>
<type>html</type>
<same-as seed="false">http://eric.van-der-vlist.com/blog/.17</same-as>
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/gallery/conferences/2002introebxml/</uri>
<local-name>eric.van-der-vlist.comindex8html.</local-name>
<local-name>eric.van-der-vlist.com/index-10.html</local-name>
<type>html</type>
<same-as seed="false">http://eric.van-der-vlist.com/blog/.20</same-as>
</resource>
@ -1677,7 +1677,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/?0:n.fx==</uri>
<local-name>eric.van-der-vlist.com/?0:n.fx==</local-name>
<local-name>eric.van-der-vlist.com/index-11.html</local-name>
<type>html</type>
</resource>
<resource>
@ -1712,7 +1712,7 @@
</resource>
<resource>
<uri seed="false">http://eric.van-der-vlist.com/blog/index.php?callback=json&amp;api_key=true&amp;format=json&amp;method=gallery&amp;id=</uri>
<local-name>eric.van-der-vlist.com/index.php?callback=json&amp;api_key=true&amp;format=json&amp;method=gallery&amp;id=</local-name>
<local-name>eric.van-der-vlist.com/index.php</local-name>
<type>json</type>
</resource>
<resource>
@ -1725,4 +1725,4 @@
<local-name>eric.van-der-vlist.com/openid.gif</local-name>
<type>gif</type>
</resource>
</index>
</index>

2
archiver/pipelines/test/local-names.xsl
View File

@ -14,7 +14,7 @@
<xsl:with-param name="id" select="uri"/>
<xsl:with-param name="nodes1">
<local-name>
<xsl:value-of select="owk:local-name(key('log-by-uri', current()/uri, $source ))"/>
<xsl:value-of select="owk:unique-local-name(key('log-by-uri', current()/uri, $source ))"/>
</local-name>
</xsl:with-param>
<xsl:with-param name="nodes2">