matagus · July 7, 2017 15:12 · Jul 5, 2017 · Jul 4, 2017 · Jul 4, 2017 · Jul 4, 2017
diff --git a/0readme.md b/0readme.md
@@ -6,6 +6,5 @@
 
 ## Usage
 
- - change the site name in the script
- - `scrapy runspider -o items.csv 1spider.py`
+ - `scrapy runspider -o items.csv -a site="https://yoursite.org" 1spider.py`
  - `python3 2format_results.py`
diff --git a/1spider.py b/1spider.py
@@ -1,17 +1,17 @@
-from scrapy.selector import HtmlXPathSelector
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
-from scrapy.item import Item, Field
+import scrapy
 
-SITE = 'https://bastamag.net'
-DOMAIN = SITE.split('//')[1]
 
-
-class BrokenLinksSpider(CrawlSpider):
+class BrokenLinksSpider(scrapy.Spider):
     name = 'brokenlink-checker'
-    start_urls = [SITE]
     handle_httpstatus_list = [404, 500]
 
+
+    def __init__(self, site, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.start_urls = [site]
+        self.DOMAIN = site.split('//')[1]
+
+
     def parse(self, response):
         if response.status in (404, 500):
             item = {}
@@ -23,7 +23,7 @@ def parse(self, response):
 
             yield item
 
-        if DOMAIN in response.url:
+        if self.DOMAIN in response.url:
             for link in response.css('a'):
                 href = link.xpath('@href').extract()
                 text = link.xpath('text()').extract()

diff --git a/0readme.md b/0readme.md
@@ -1,3 +1,5 @@
+# List all the broken links on your website
+
 ## Requirements:
 
 `python3` and `scrapy` (`pip install scrapy`)

diff --git a/0readme.md b/0readme.md
@@ -0,0 +1,9 @@
+## Requirements:
+
+`python3` and `scrapy` (`pip install scrapy`)
+
+## Usage
+
+ - change the site name in the script
+ - `scrapy runspider -o items.csv 1spider.py`
+ - `python3 2format_results.py`
diff --git a/spider.py → 1spider.py b/spider.py → 1spider.py
diff --git a/format_results.py → 2format_results.py b/format_results.py → 2format_results.py
@@ -1,12 +1,11 @@
 import csv, itertools
 
 items = csv.DictReader(open('items.csv'))
-for page, links in itertools.groupby(items, lambda item: item['prev_page']):    
-    print('PAGE:', page)
-    for line in links:
-        if line['prev_page']:
+for page, links in itertools.groupby(items, lambda item: item['prev_page']):
+    if page:
+        print('PAGE:', page)
+        for line in links:
             print('     LINK TEXT:', line['prev_link_text'])
             print('     LINK URL:', line['prev_link_url'])
-            print('     LINK STATUS:', line['status'])
             print()
-    print()
+        print()
diff --git a/format_results.py b/format_results.py
@@ -0,0 +1,12 @@
+import csv, itertools
+
+items = csv.DictReader(open('items.csv'))
+for page, links in itertools.groupby(items, lambda item: item['prev_page']):    
+    print('PAGE:', page)
+    for line in links:
+        if line['prev_page']:
+            print('     LINK TEXT:', line['prev_link_text'])
+            print('     LINK URL:', line['prev_link_url'])
+            print('     LINK STATUS:', line['status'])
+            print()
+    print()
diff --git a/spider.py b/spider.py
@@ -0,0 +1,35 @@
+from scrapy.selector import HtmlXPathSelector
+from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.item import Item, Field
+
+SITE = 'https://bastamag.net'
+DOMAIN = SITE.split('//')[1]
+
+
+class BrokenLinksSpider(CrawlSpider):
+    name = 'brokenlink-checker'
+    start_urls = [SITE]
+    handle_httpstatus_list = [404, 500]
+
+    def parse(self, response):
+        if response.status in (404, 500):
+            item = {}
+            item['url'] = response.url
+            item['prev_page'] = response.meta['prev_url']
+            item['prev_link_url'] = response.meta['prev_href']
+            item['prev_link_text'] = response.meta['prev_link_text']
+            item['status'] = response.status
+
+            yield item
+
+        if DOMAIN in response.url:
+            for link in response.css('a'):
+                href = link.xpath('@href').extract()
+                text = link.xpath('text()').extract()
+                if href: # maybe should show an error if no href
+                    yield response.follow(link, self.parse, meta={
+                        'prev_link_text': text,
+                        'prev_href': href,
+                        'prev_url': response.url,
+                    })