Created
February 14, 2020 01:24
-
-
Save binarybrat/003f739eaac4371348b44afde70b0ecb to your computer and use it in GitHub Desktop.
my comment model
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import tldextract | |
| from datetime import datetime, timedelta | |
| from models import BaseModel | |
| from peewee import * | |
| from models.blah import YoutubeLogging, Tutorials | |
| import time | |
| from reddit.utils.globalvars import * | |
| from reddit.utils.randomhelpershit import * | |
| from reddit.utils._company_rgx import * | |
| from models.detailsmodel import * | |
| from reddit.tubeChecking import * | |
| tutorialsfromcomments = [] | |
| tubelinks = [] | |
| class Comment(BaseModel): | |
| approved_by = TextField(null=True) | |
| archived = IntegerField(null=True) | |
| author = CharField() | |
| banned_by = TextField(null=True) | |
| body = TextField(null=True) | |
| created = DateTimeField() | |
| edited = DateTimeField(null=True) | |
| distinguished = CharField(null=True) | |
| posflair = CharField(null=True) | |
| fullname = CharField() | |
| gilded = CharField(null=True) | |
| idstr = CharField(unique=True) | |
| ignore_reports = IntegerField(null=True) | |
| is_root = IntegerField() | |
| is_submitter = IntegerField() | |
| link_author = CharField(null=True) | |
| link_title = TextField(null=True) | |
| link_permalink = TextField(null=True) | |
| link_url = TextField(null=True) | |
| spam = CharField(null=True) | |
| submission = CharField(null=True) | |
| parent = CharField(null=True) | |
| permalink = TextField(null=True) | |
| removed = IntegerField(null=True) | |
| score = IntegerField(null=True) | |
| stickied = IntegerField() | |
| subreddit = CharField(null=True) | |
| ups = IntegerField() | |
| perspective = TextField(null=True) | |
| ureports_dismissed = TextField(null=True) | |
| fetched = DateTimeField(constraints=[SQL("DEFAULT CURRENT_TIMESTAMP")], null=True) | |
| class Meta: | |
| table_name = 'Comment' | |
| class CommentEdits(BaseModel): | |
| idstr = CharField() | |
| text = TextField(null=True) | |
| replaced_at = DateTimeField() | |
| subreddit = CharField() | |
| class Meta: | |
| table_name = 'CommentEdits' | |
| indexes = ( | |
| (('idstr', 'replaced_at'), True), | |
| ) | |
| primary_key = CompositeKey('idstr', 'replaced_at') | |
| def lastComment(): | |
| commentsDB = Comment.select() | |
| test = [pos.created for pos in commentsDB.order_by(Comment.created.desc()).limit(1)] | |
| return time.mktime(test[0].timetuple()) | |
| def cleancomlinks(lisofcomlinks): | |
| newlinklist = [] | |
| for comlink in lisofcomlinks: | |
| link = comlink.split("</a>")[0] if "</a>" in comlink else comlink | |
| if link not in newlinklist: | |
| newlinklist.append(link) | |
| return newlinklist | |
| def comDone(commentobj): | |
| comdone = Comment.get_or_none(Comment.fullname == commentobj) | |
| return True if comdone else False | |
| class CommentHelper: | |
| def __init__(self, reddit, comment, grablinks=False, dammit=False, verbose=False, overkill=False, wordmin=50): | |
| self.verbose = verbose | |
| self.overkill = overkill | |
| self.grablinks = grablinks | |
| self.reddit = reddit | |
| self.comment = comment | |
| self.permalink = 'https://www.reddit.com' + self.comment.permalink | |
| self.wordmin = wordmin | |
| self.wordcount = len(self.comment.body.split()) | |
| self.submission = self.reddit.submission(self.comment.submission.id) | |
| self.link_author = self.submission.author if self.submission.author is not None else '[deleted]' | |
| self.link_title = self.submission.title | |
| self.link_permalink = 'https://www.reddit.com' + self.submission.permalink | |
| self.link_url = self.submission.url | |
| self.link_score = self.submission.score | |
| self.link_flair = self.submission.link_flair_text if self.submission.link_flair_text is not None else None | |
| self.is_deleted = True if (self.comment.author is None and self.comment.body == '[deleted]') else False | |
| self.submission_deleted = True if self.submission.author is None or self.comment.body == '[deleted]' else False | |
| self.pattern_comment = True if ( | |
| re.search(bcomprgx, self.comment.body) or re.search(newergx, self.comment.body) or re.search( | |
| burdargx, self.comment.body)) else False | |
| self.tutorial_comment = True if (re.search(r"(tutorial[s]?|free pattern[s]?)", self.comment.body, | |
| re.IGNORECASE) and 'http' in self.comment.body.lower()) else False | |
| self.details_comment = True if not isdeleted(self.comment) and not self.comment.spam and not self.comment.removed and not isdeleted( | |
| self.submission) and not self.submission.spam and not (self.submission.removed and self.submission.banned_by is not None and self.submission.banned_by != 'sewingmodthings') and self.comment.is_submitter and self.comment.parent_id.startswith('t3_') and self.link_flair is not None and self.link_flair.lower() in [x.lower() for x in | |
| FLAIRTOCHECK] and ( | |
| self.pattern_comment or self.wordcount >= self.wordmin) else False | |
| self.is_rising = True if self.submission.score >= 900 else False | |
| self.previewimg = self.submission.preview['images'][0]['resolutions'][2]['url'] if self.details_comment and has_preview(self.submission) else None | |
| self.has_comlinks = False | |
| self.low_comment = False | |
| self.dammit = dammit | |
| if self.dammit: | |
| self.doitdammit() | |
| if self.grablinks and not isdeleted( | |
| self.comment) and not self.comment.spam and not self.comment.removed: | |
| self.grab_commentlinks() | |
| if self.commentlinks: | |
| self.has_comlinks = True | |
| def doitdammit(self): | |
| self.existing = Comment.get_or_none(Comment.idstr == self.comment.id) | |
| if not self.existing: | |
| self.add_comment() | |
| else: | |
| if self.comment.author is None and self.comment.body == '[deleted]': | |
| self.updateDeleted_comment() | |
| else: | |
| self.update_comment() | |
| def add_comment(self): | |
| Comment.create( | |
| approved_by=self.comment.approved_by if self.comment.approved_by is None else None, | |
| archived=self.comment.archived, | |
| author=self.comment.author.name if self.comment.author else "[deleted]", | |
| banned_by=self.comment.banned_by if self.comment.banned_by is None else None, | |
| created=datetime.fromtimestamp(self.comment.created_utc), | |
| distinguished=self.comment.distinguished, | |
| fullname=self.comment.fullname, | |
| gilded=self.comment.gilded, | |
| idstr=self.comment.id, | |
| posflair=self.link_flair, | |
| is_root=self.comment.is_root, | |
| is_submitter=self.comment.is_submitter, | |
| link_author=self.link_author, | |
| link_title=self.link_title, | |
| link_permalink=self.link_permalink, | |
| link_url=self.link_url, | |
| removed=self.comment.removed, | |
| spam=self.comment.spam, | |
| submission=self.comment.submission.id, | |
| parent=None if self.comment.is_root else self.comment.parent_id[3:], | |
| permalink=self.comment.permalink, | |
| subreddit=self.comment.subreddit.display_name, | |
| perspective=None, | |
| body=self.comment.body, | |
| edited=datetime.fromtimestamp(self.comment.edited) if self.comment.edited else None, | |
| score=self.comment.score, | |
| stickied=self.comment.stickied, | |
| ups=self.comment.ups, | |
| ureports_dismissed=self.comment.user_reports_dismissed if hasattr(self.comment, | |
| 'user_reports_dismissed') else None | |
| ) | |
| if self.verbose: | |
| print("Added comment: {} by u/{}".format(self.comment.fullname, self.comment.author.name if | |
| self.comment.author is not None else '[deleted]')) | |
| def update_comment(self): | |
| toupdate = Comment.update( | |
| approved_by=self.comment.approved_by if self.comment.approved_by is not None else None, | |
| archived=self.comment.archived, | |
| banned_by=self.comment.banned_by, | |
| body=self.comment.body, | |
| edited=datetime.fromtimestamp(self.comment.edited) if self.comment.edited else None, | |
| distinguished=self.comment.distinguished, | |
| posflair=self.link_flair, | |
| gilded=self.comment.gilded, | |
| ignore_reports=self.comment.ignore_reports, | |
| removed=self.comment.removed, | |
| spam=self.comment.spam, | |
| score=self.comment.score, | |
| stickied=self.comment.stickied, | |
| ups=self.comment.ups, | |
| ureports_dismissed=self.comment.user_reports_dismissed if hasattr(self.comment, 'user_reports_dismissed') else None).where(Comment.idstr == self.comment.id) | |
| toupdate.execute() | |
| if self.overkill: | |
| return print("Updated comment: {}".format(self.comment.fullname)) | |
| def updateDeleted_comment(self): | |
| todelupdate = Comment.update( | |
| approved_by=self.comment.approved_by if self.comment.approved_by is not None else None, | |
| archived=self.comment.archived, | |
| banned_by=self.comment.banned_by, | |
| edited=datetime.fromtimestamp(self.comment.edited) if self.comment.edited else None, | |
| distinguished=self.comment.distinguished, | |
| posflair=self.link_flair, | |
| gilded=self.comment.gilded, | |
| ignore_reports=self.comment.ignore_reports, | |
| removed=self.comment.removed, | |
| spam=self.comment.spam, | |
| score=self.comment.score, | |
| stickied=self.comment.stickied, | |
| ups=self.comment.ups, | |
| ureports_dismissed=self.comment.user_reports_dismissed if | |
| hasattr(self.comment, 'user_reports_dismissed') else None).where(Comment.idstr == self.comment.id) | |
| todelupdate.execute() | |
| if self.verbose: | |
| print("Updated DELETED comment: {}".format(self.comment.fullname)) | |
| def insert_edited(self, old_text): | |
| if self.comment.edited is False: | |
| replaced_at = datetime.now() | |
| else: | |
| replaced_at = datetime.fromtimestamp(self.comment.edited) | |
| try: | |
| CommentEdits.create(idstr=self.comment.fullname, replaced_at=replaced_at, text=old_text, | |
| subreddit=self.comment.subreddit.display_name) | |
| if self.verbose: | |
| return print("Inserted Edited Comment For: {}".format(self.comment.fullname)) | |
| except IntegrityError: | |
| pass | |
| def already_in_db(self): | |
| self.cdone = Comment.get_or_none(Comment.idstr == self.comment.id) | |
| return True if self.cdone is not None else False | |
| def check_for_pattern(self): | |
| if re.search(burdargx, self.comment.body) or re.search(bcomprgx, self.comment.body) or re.search(newergx, | |
| self.comment.body): | |
| self.pattern_comment = True | |
| def check_for_tutorial(self): | |
| if re.search(r"(tutorial[s]?|free pattern[s]?)", self.comment.body, re.IGNORECASE): | |
| self.tutorial_comment = True | |
| def check_for_details(self): | |
| if not isdeleted(self.comment) and not self.comment.spam and not self.comment.removed: | |
| if self.comment.is_submitter and self.comment.parent_id.startswith( | |
| 't3_') and self.link_flair is not None and self.link_flair.lower() in [x.lower() for x in | |
| FLAIRTOCHECK]: | |
| if self.pattern_comment or len(self.comment.body.split()) >= self.wordmin: | |
| self.createdat = datetime.utcfromtimestamp(self.comment.created_utc) | |
| self.details_comment = True | |
| def grab_commentlinks(self): | |
| self.commentlinks = findhtml(self.comment.body_html) | |
| if self.commentlinks: | |
| self.has_comlinks = True | |
| def low_comment_check(self): | |
| if self.comment.score <= -5 and self.comment.approved_by is None and self.comment.banned_by is None and not \ | |
| self.comment.removed and self.comment.author is not None: | |
| self.low_comment = True | |
| def force_details(self): | |
| fupdate = DetailsPending.update(ignore=True).where(DetailsPending.idstr == self.submission.fullname) | |
| fupdate.execute() | |
| print("Forced Pending Details to approved since a mod already approved the post.") | |
| def approve_details_silent(self): | |
| return True if time.time() <= 1565762849.0 else False | |
| def grab_author_backcoms(reddit, authorname, verbose=False): | |
| count = 0 | |
| processed = 0 | |
| delupdated = 0 | |
| updated = 0 | |
| for comment in reddit.redditor(authorname).comments.new(limit=None): | |
| count += 1 | |
| if count % 300 == 0: | |
| print(count, 'done') | |
| if comment.subreddit.display_name == 'sewing' and not isdeleted(comment): | |
| try: | |
| chelper = CommentHelper(reddit, comment, grablinks=True, verbose=verbose) | |
| if not chelper.already_in_db(): | |
| chelper.add_comment() | |
| processed += 1 | |
| if chelper.has_comlinks and chelper.comment.author != 'sewingmodthings' and ( | |
| chelper.link_flair is None or chelper.link_flair is not None and chelper.link_flair.lower() not in | |
| ['machine questions', 'suggest machine', 'fabric question']): | |
| commentlinks = cleancomlinks(chelper.commentlinks) | |
| for linkurl in commentlinks: | |
| if re.search(tubergx, linkurl) and ("/channel/" or "/user/") not in linkurl: | |
| process_tube_data(chelper.comment, linkurl) | |
| time.sleep(5) | |
| else: | |
| ext = tldextract.extract(linkurl) | |
| if not tutDone(chelper.comment.fullname, linkurl) and not ( | |
| ext.domain == 'spoonflower' and ext.subdomain == 'www') and not re.search( | |
| parserdomainsrgx, ext.domain) and not re.search(domainstoignore, | |
| ext.domain) and not re.search( | |
| isfilergx, linkurl): | |
| try: | |
| linktitle = grab_tutorial_info(chelper.comment.fullname, linkurl, verbose=True) | |
| add_tutorial(chelper.comment, tags=None, linktitle=linktitle, linkurl=linkurl, | |
| verbose=True) | |
| time.sleep(5) | |
| except IntegrityError: | |
| pass | |
| except Exception as e: | |
| pass | |
| else: | |
| if chelper.is_deleted: | |
| chelper.updateDeleted_comment() | |
| delupdated += 1 | |
| else: | |
| chelper.update_comment() | |
| updated += 1 | |
| except IndexError: | |
| pass | |
| print("Comments >> Processed: {} || Updated: {}".format(processed, updated)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment