from scrapy import Request class SynchronousSpiderMiddleware(object): def process_spider_output(self, response, result, spider): listing_request = None for request in result: step = request.meta.get('step', None) if step == 'parse': if not listing_request: listing_request = request listings = listing_request.meta.get('listings', []) listings.append(request) continue elif step == 'result': listings = request.meta.get('listings', []) request = listings.pop(0) yield request if step == 'parse': yield listing_request