diff --git a/sitemap/sitemap.sh b/sitemap/sitemap.sh deleted file mode 100644 index e03363a..0000000 --- a/sitemap/sitemap.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# sitemap.sh by heuzef.com (CC-BY-NC) -# Usage : sh sitemap.sh www.heuzef.com - -echo "Generating ..." -echo -sitedomain=$1 -wget --spider --recursive --level=inf --no-verbose --output-file=get_urls.txt $sitedomain -grep -i URL get_urls.txt | awk -F 'URL:' '{print $2}' | awk '{$1=$1};1' | awk '{print $1}' | sort -u | sed '/^$/d' > urls_$sitedomain.txt -header='' -echo $header > sitemap_$sitedomain.xml -while read p; do - case "$p" in - */ | *.html | *.htm) - echo ''$p'' >> sitemap_$sitedomain.xml - ;; - *) - ;; - esac -done < urls_$sitedomain.txt -echo "" >> sitemap_$sitedomain.xml -rm get_urls.txt -rm -r $sitedomain -echo "Sitemap generated OK :" -echo -ls -larth | grep $sitedomain diff --git a/sitemap/sitemap_example.com.xml b/sitemap/sitemap_example.com.xml deleted file mode 100644 index d32d456..0000000 --- a/sitemap/sitemap_example.com.xml +++ /dev/null @@ -1,3 +0,0 @@ - -http://example.com/ - diff --git a/sitemap/sitemap_www.aoc-toiture.com.xml b/sitemap/sitemap_www.aoc-toiture.com.xml deleted file mode 100644 index 7bee033..0000000 --- a/sitemap/sitemap_www.aoc-toiture.com.xml +++ /dev/null @@ -1,3 +0,0 @@ - -http://www.aoc-toiture.com/ - diff --git a/sitemap/sitemap_www.firewall-services.com.xml b/sitemap/sitemap_www.firewall-services.com.xml deleted file mode 100644 index 313ef84..0000000 --- a/sitemap/sitemap_www.firewall-services.com.xml +++ /dev/null @@ -1,12 +0,0 @@ - -http://www.firewall-services.com/ -http://www.firewall-services.com/clubs-entreprises.html -http://www.firewall-services.com/contact.html -http://www.firewall-services.com/espace-presse.html -http://www.firewall-services.com/formations.html -http://www.firewall-services.com/index.html -http://www.firewall-services.com/nos-competences.html -http://www.firewall-services.com/passeport.html -http://www.firewall-services.com/plan.html -http://www.firewall-services.com/prolibre.html - diff --git a/urls/slurp_urls.sh b/urls/slurp_urls.sh new file mode 100644 index 0000000..f5fcd65 --- /dev/null +++ b/urls/slurp_urls.sh @@ -0,0 +1,11 @@ +#!/bin/bash +echo "Slurping URLs ..." +echo +sitedomain=$1 +wget --spider --recursive --level=inf --no-verbose --output-file=get_urls.txt $sitedomain +grep -i URL get_urls.txt | awk -F 'URL:' '{print $2}' | awk '{$1=$1};1' | awk '{print $1}' | sort -u | sed '/^$/d' > urls_$sitedomain.txt +rm get_urls.txt +rm -r $sitedomain +echo "Done !" +echo +ls -lArth | grep urls_$sitedomain.txt diff --git a/urls/urls_www.perl.org.txt b/urls/urls_www.perl.org.txt new file mode 100644 index 0000000..34a3071 --- /dev/null +++ b/urls/urls_www.perl.org.txt @@ -0,0 +1,107 @@ +https://www.perl.org/ +https://www.perl.org/about.html +https://www.perl.org/about/reduce-risks.html +https://www.perl.org/about/style-guide.html +https://www.perl.org/about/whitepapers/ +https://www.perl.org/about/whitepapers/perl-cpan.html +https://www.perl.org/about/whitepapers/perl-database-orm.html +https://www.perl.org/about/whitepapers/perl-ipv6.html +https://www.perl.org/about/whitepapers/perl-loadbalancer.html +https://www.perl.org/about/whitepapers/perl-object-oriented.html +https://www.perl.org/about/whitepapers/perl-plack.html +https://www.perl.org/about/whitepapers/perl-profiling.html +https://www.perl.org/about/whitepapers/perl-testing.html +https://www.perl.org/about/whitepapers/perl-webframework.html +https://www.perl.org/advocacy/white_camel/ +https://www.perl.org/advocacy/white_camel/1999.html +https://www.perl.org/advocacy/white_camel/2000.html +https://www.perl.org/advocacy/white_camel/2001.html +https://www.perl.org/advocacy/white_camel/2002.html +https://www.perl.org/advocacy/white_camel/2003.html +https://www.perl.org/advocacy/white_camel/2004.html +https://www.perl.org/advocacy/white_camel/2005.html +https://www.perl.org/advocacy/white_camel/2006.html +https://www.perl.org/advocacy/white_camel/2007.html +https://www.perl.org/advocacy/white_camel/2008.html +https://www.perl.org/advocacy/white_camel/2009.html +https://www.perl.org/advocacy/white_camel/2010.html +https://www.perl.org/advocacy/white_camel/2011.html +https://www.perl.org/advocacy/white_camel/2012.html +https://www.perl.org/advocacy/white_camel/2013.html +https://www.perl.org/advocacy/white_camel/2014.html +https://www.perl.org/advocacy/white_camel/2015.html +https://www.perl.org/advocacy/white_camel/2016.html +https://www.perl.org/advocacy/white_camel/2017.html +https://www.perl.org/app.html +https://www.perl.org/books/beginning-perl/ +https://www.perl.org/books/impatient-perl/ +https://www.perl.org/books/library.html +https://www.perl.org/camel.html +https://www.perl.org/community.html +https://www.perl.org/contribute.html +https://www.perl.org/cpan.html +https://www.perl.org/docs.html +https://www.perl.org/events.html +https://www.perl.org/get.html +https://www.perl.org/learn.html +https://www.perl.org/media.html +https://www.perl.org/robots.txt +https://www.perl.org/siteinfo.html +https://www.perl.org/wishlist.html +http://www.perl.org/ +http://www.perl.org/about.html +http://www.perl.org/about/reduce-risks.html +http://www.perl.org/about/style-guide.html +http://www.perl.org/about/whitepapers/ +http://www.perl.org/about/whitepapers/perl-cpan.html +http://www.perl.org/about/whitepapers/perl-database-orm.html +http://www.perl.org/about/whitepapers/perl-ipv6.html +http://www.perl.org/about/whitepapers/perl-loadbalancer.html +http://www.perl.org/about/whitepapers/perl-object-oriented.html +http://www.perl.org/about/whitepapers/perl-plack.html +http://www.perl.org/about/whitepapers/perl-profiling.html +http://www.perl.org/about/whitepapers/perl-testing.html +http://www.perl.org/about/whitepapers/perl-webframework.html +http://www.perl.org/advocacy/white_camel/ +http://www.perl.org/advocacy/white_camel/1999.html +http://www.perl.org/advocacy/white_camel/2000.html +http://www.perl.org/advocacy/white_camel/2001.html +http://www.perl.org/advocacy/white_camel/2002.html +http://www.perl.org/advocacy/white_camel/2003.html +http://www.perl.org/advocacy/white_camel/2004.html +http://www.perl.org/advocacy/white_camel/2005.html +http://www.perl.org/advocacy/white_camel/2006.html +http://www.perl.org/advocacy/white_camel/2007.html +http://www.perl.org/advocacy/white_camel/2008.html +http://www.perl.org/advocacy/white_camel/2009.html +http://www.perl.org/advocacy/white_camel/2010.html +http://www.perl.org/advocacy/white_camel/2011.html +http://www.perl.org/advocacy/white_camel/2012.html +http://www.perl.org/advocacy/white_camel/2013.html +http://www.perl.org/advocacy/white_camel/2014.html +http://www.perl.org/advocacy/white_camel/2015.html +http://www.perl.org/advocacy/white_camel/2016.html +http://www.perl.org/advocacy/white_camel/2017.html +http://www.perl.org/app.html +http://www.perl.org/books/beginning-perl/ +http://www.perl.org/books/impatient-perl/ +http://www.perl.org/books/library.html +http://www.perl.org/camel.html +http://www.perl.org/community.html +http://www.perl.org/contribute.html +http://www.perl.org/cpan.html +http://www.perl.org/docs.html +http://www.perl.org/events.html +http://www.perl.org/get.html +http://www.perl.org/learn.html +http://www.perl.org/media.html +http://www.perl.org/robots.txt +http://www.perl.org/siteinfo.html +http://www.perl.org/wishlist.html +http://www.powells.com/book/beginning-perl-9781861003140?partnerid=25774 +http://www.powells.com/book/embedding-perl-in-html-with-mason-9780596002251?partnerid=25774 +http://www.powells.com/book/modern-perl-9781680500882?partnerid=25774 +http://www.powells.com/book/modperl-developers-cookbook-9780672322402?partnerid=25774 +http://www.powells.com/book/perl-lwp-9780596001780?partnerid=25774 +http://www.powells.com/book/practical-mod-perl-9780596002275?partnerid=25774 +http://www.powells.com/book/writing-apache-modules-with-perl-and-c-9781565925670X?partnerid=25774