The script starts by pulling an existing article archive CSV file and checks every new Invoke-Webrequest article against it. It then strips the duplicates and processes the new articles. After it's done processing and formatting the articles' output, it checks to see if there are at least 15 new articles. If so, continues with the script while adding the new articles to the archive. Then the script finds the most common title words, groups the articles based on those top 10 keywords, and begins formatting each article for its HTML section. Once completed, then emails me the completed article.
I've cut the script into several sections because if its length. This way you can scroll horizontally as needed without having to go to the bottom of the page. I've also included an example of the output created by this script. Open it with a browser to see how it looks. It may not suit everyone but I wanted a down and dirty tech/news emailer without the ads and occasional over-sized pictures. Plus, I can keep up on new events as they occur on an hourly basis.
$TheList = $nul $NewOnly = $nul $Articles = $nul $a = $nul $MasterNumber = 0 $iwList = $nul $FinalList = $nul if (test-path \\server\share\scripts\RSSMailerArchive.csv){$Archive = import-csv \\server\share\scripts\RSSMailerArchive.csv} #### TECHSPOT #### [xml]$techspot = Invoke-Webrequest https://www.techspot.com/backend.xml $articles = $techspot.getelementsbytagname('item') #Removing title because it's stuck as an XML element and I need it to be a string one. foreach ($a in $articles){ $a|add-member -notepropertyname "temp" -notepropertyvalue $a.title.'#cdata-section' } $articles = $articles|select temp,creator,pubdate,link,guid,description #Add a string version of Title and moving from temp to title. foreach ($a in $articles){ $a|add-member -notepropertyname "title" -notepropertyvalue $a.temp } #stripping out temp $articles = $articles|select title,creator,pubdate,link,guid,description #removing duplicate articles already listed in the archive $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ if ([datetime]$a.pubdate -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$a.pubdate).tooadate() $a|add-member -notepropertyname "Author" -notepropertyvalue $a.creator.'#cdata-section' $link = $a.link $title = $a.title.trim() $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $html = New-Object -ComObject "HTMLFile" $html.IHTMLDocument2_write($a.description.'#cdata-section') $imglink = $html.body.getelementsbytagname('img')[0].src $text = $html.body.innertext.trim() $a|add-member -notepropertyname "Textonly" -notepropertyvalue $text $a|add-member -notepropertyname "Source" -notepropertyvalue ("TechSpot " + ([datetime]$a.pubdate).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;'><a name=$index href=$link><img width=200 vspace=5 hspace=5 align=left src=" + $imglink + "></a><h3>" + $title + "</h3></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $text + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ [System.Runtime.Interopservices.Marshal]::ReleaseComObject($html)|out-null } #### END TECHSPOT ####
#### ANDROID CENTRAL #### [xml]$ac = Invoke-Webrequest http://feeds2.feedburner.com/androidcentral $articles = $ac.getelementsbytagname('item') $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ if ([datetime]$a.pubdate -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$a.pubdate).tooadate() $title = $a.title.trim() $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname "Author" -notepropertyvalue $a.creator $link = $a.link $html = New-Object -ComObject "HTMLFile" $encoded = ($a.encoded.'#cdata-section') $html.IHTMLDocument2_write($encoded) $text = $html.body.innerhtml if ($text -ne $nul){$text = $text.replace("<IMG","<img width=300 align=left hspace=10 vspace=10 ")} if ($text -eq $nul){ $text = $a.encoded $text = $text -replace ('<style>+','~') $text = $text.split('~')[0] $text = $text.replace("<IMG","<IMG width=300 align=left hspace=10 vspace=10 ") $text = $text.replace("<img","<img width=300 align=left hspace=10 vspace=10 ") } $text = $text.trim() $a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.description $a|add-member -notepropertyname "Source" -notepropertyvalue ("Android Central " + ([datetime]$a.pubdate).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width=100%><tr><td><a name=$index></a><h3>$title</h3></td></tr><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $text + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$thelist += $a} $MasterNumber++ [System.Runtime.Interopservices.Marshal]::ReleaseComObject($html)|out-null } #### END ANDROID CENTRAL ####
#### ANDROID POLICE #### [xml]$ap = Invoke-Webrequest http://www.androidpolice.com/feed/ $articles = $ap.getelementsbytagname('item') $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ if ([datetime]$a.pubdate -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$a.pubdate).tooadate() $link = $a.link $comments = ($a.link + "#comments") $title = $a.title.trim() $title = $title -replace ('\[','"') $title = $title -replace ('\]','"') $title = $title -replace ("\?\?\?",'"') $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title.trim() $a|add-member -notepropertyname "Author" -notepropertyvalue $a.creator.'#cdata-section' $html = New-Object -ComObject "HTMLFile" $description = ($a.description.'#cdata-section') $html.IHTMLDocument2_write($description) $imglink = $html.body.getelementsbytagname('img')[0].src $text = $html.body.innertext $text = $text -replace ('\[','"') $text = $text -replace ('\]','"') $text = $text -replace ('Read More\s+') $text = $text -replace ($title) $text = $text.replace(" was written by the awesome team at Android Police.","") $text = $text.trim() $text = $text + " <a href=$comments>Comments</a>" $a|add-member -notepropertyname "Textonly" -notepropertyvalue $text $a|add-member -notepropertyname "Source" -notepropertyvalue ("Android Police " + ([datetime]$a.pubdate).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;'><a name=$index href=$link><img vspace=5 hspace=5 width=200 align=left src=" + $imglink + "></a><h3>" + $title + "</h3></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $text + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$thelist += $a} $MasterNumber++ [System.Runtime.Interopservices.Marshal]::ReleaseComObject($html)|out-null } #### END ANDROID POLICE ####
#### SLASHDOT #### [xml]$slashdot = Invoke-Webrequest http://rss.slashdot.org/Slashdot/slashdotMain $articles = $slashdot.getelementsbytagname('item') $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $link = $a.link $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title.trim() [string]$a.date = ([datetime]$a.date).tooadate() $a|add-member -notepropertyname "Author" -notepropertyvalue $a.creator $comments = $a.comments $department = $a.department $description = $a.description $subjecticon = ("<img align=left vspace=5 hspace=5 src=https://a.fsdn.com/sd/topics/" + $a.subject + "_64.png>") $html = New-Object -ComObject "HTMLFile" $html.IHTMLDocument2_write($a.description) $text = $html.body.innertext $text = $text.replace('Read more of this story at Slashdot.','') $a|add-member -notepropertyname "Textonly" -notepropertyvalue $text $a|add-member -notepropertyname "Source" -notepropertyvalue ("Slashdot " + ([datetime]::FromOADate($a.date))) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table style='width:100%'><tr><td style='font-family: Arial, sans-serif;'><h3><a name=$index href=$link>" + $subjecticon + "</a>" + $title + "</h3><b>" + $comments + "</b> comments <b><i>(" + $department + ")</b></i></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $text + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$TheList +=$a} $MasterNumber++ [System.Runtime.Interopservices.Marshal]::ReleaseComObject($html)|out-null } #### END SLASHDOT ####
#### PHONE ARENA #### [xml]$phonearena = Invoke-Webrequest "https://www.phonearena.com/feed" $articles = $phonearena.rss.channel.item $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ if ([datetime]$a.pubdate -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$a.pubdate).tooadate() $link = $a.link $title = $a.title $title = $title.replace("???",'"') $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title.trim() $html = New-Object -ComObject "HTMLFile" $getimglink = ($a.description.'#cdata-section') $html.IHTMLDocument2_write($getimglink) $imglink = $html.body.getelementsbytagname('img')|select -expand src $text = $html.body.innertext $text = $text.replace("???",'"') $a|add-member -notepropertyname "Textonly" -notepropertyvalue $text.trim() $a|add-member -notepropertyname "Source" -notepropertyvalue ("Phone Arena " + ([datetime]$a.pubdate).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;'><a name=$index href=$link><img width=200 align=left vspace=5 hspace=5 src=" + $imglink + "></a><h3>" + $title + "</h3></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $text + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$TheList +=$a} $MasterNumber++ [System.Runtime.Interopservices.Marshal]::ReleaseComObject($html)|out-null } #### END PHONE ARENA ####
#### UBERGIZMO #### [xml]$uber = Invoke-Webrequest http://feeds.feedburner.com/ubergizmo $articles = $uber.GetElementsByTagName('item') $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ if ([datetime]$a.pubdate -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$a.pubdate).tooadate() $title = $a.title $title = $title.replace("???","'") $title = $title.replace("??",'"') $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title.trim() $a|add-member -notepropertyname "Author" -notepropertyvalue $a.creator $link = $a.link $html = New-Object -ComObject "HTMLFile" $article = $a.encoded.'#cdata-section' $html.IHTMLDocument2_write($article) $imglink = $html.body.getelementsbytagname('img')[0].src $text = $html.body.innertext #Getting rid of article spam at bottom of each article $text = $text.replace("???","'").replace("??",'"').replace($title,'') $text = $text.replace(', original content from Ubergizmo. Read our Copyrights and terms of use.','') $a|add-member -notepropertyname "Textonly" -notepropertyvalue $text.trim() $a|add-member -notepropertyname "Source" -notepropertyvalue ("Ubergizmo " + ([datetime]$a.pubdate).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style=font-family: Arial, sans-serif;><a name=$index href=$link><img vspace=5 hspace=5 width=200 align=left src=" + $imglink + "></a><h3>" + $title + "</h3></td></tr></table><table width='100%'><tr><td style='font-family: Arial, sans-serif;font-size:13;'><p>" + $text + "</p></td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$TheList +=$a} $MasterNumber++ [System.Runtime.Interopservices.Marshal]::ReleaseComObject($html)|out-null } #### END UBERGIZMO ####
#### IEEE SPECTRUM #### [xml]$ieee = Invoke-Webrequest http://www.spectrum.ieee.org/rss $articles = $ieee.getelementsbytagname('item') $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ if ([datetime]$a.pubdate -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$a.pubdate).tooadate() $title = $a.title $title = $title.replace("???","'") $title = $title.replace("??",'"') $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title.trim() $a|add-member -notepropertyname "Author" -notepropertyvalue $a.creator $link = $a.link $a|add-member -notepropertyname "Source" -notepropertyvalue ("IEEE " + ([datetime]$a.pubdate).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $encoded = $a.encoded.replace("<a","<a name=$index") $imgandtext = $encoded.replace("<img","<img vspace=5 hspace=5 width=200 align=left") $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style=font-family: Arial, sans-serif;>$imgandtext</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$TheList +=$a} $MasterNumber++ } #### END IEEE SPECTRUM ####
#### WIRED #### [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 [xml]$WiredGear = Invoke-Webrequest https://www.wired.com/feed/category/gear/latest/rss $Articles = $WiredGear.rss.channel.getelementsbytagname('item') [xml]$WiredBusiness = Invoke-Webrequest https://www.wired.com/feed/category/business/latest/rss $Articles += $WiredBusiness.rss.channel.getelementsbytagname('item') [xml]$WiredCulture = Invoke-Webrequest https://www.wired.com/feed/category/culture/latest/rss $Articles += $WiredCulture.rss.channel.getelementsbytagname('item') [xml]$WiredScience = Invoke-Webrequest https://www.wired.com/feed/category/science/latest/rss $Articles += $WiredScience.rss.channel.getelementsbytagname('item') [xml]$WiredSecurity = Invoke-Webrequest https://www.wired.com/feed/category/security/latest/rss $Articles += $WiredSecurity.rss.channel.getelementsbytagname('item') $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ if ([datetime]$a.pubdate -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$a.pubdate).tooadate() $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title.trim() $a|add-member -notepropertyname "Author" -notepropertyvalue $a.creator $link = $a.link $a|add-member -notepropertyname "Source" -notepropertyvalue ("Wired " + ([datetime]$a.pubdate).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $article = Invoke-Webrequest $link $text = $article.parsedhtml.body.getElementsByTagName('p')[0].innertext $a|add-member -notepropertyname "Textonly" -notepropertyvalue $text.trim() $img = $article.parsedhtml.body.getelementsbytagname('img')[0].src $imgsections = $img.split('/') $imgsection = $imgsections[$imgsections.count - 2] $img = $img.replace($imgsection,'w_200,c_limit') $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style=font-family: Arial, sans-serif;><a name=$index href=$link><img vspace=5 hspace=5 align=left src=" + $img + "></a><h3>" + $title + "</h3></td></tr></table><table width='100%'><tr><td style='font-family: Arial, sans-serif;font-size:13;'><p>" + $text + "</p></td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$TheList +=$a} $MasterNumber++ } #### END WIRED ####
#### Reddit PowerShell #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/powershell/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ #Reddit uses the Unix Epoch date for their [datetime] properties $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $link = $a.url $comments = $a.num_comments $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl $score = $a.score $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} if ($a.selftext.length -eq 0){ $getarticle = Invoke-Webrequest $a.url $innertext = $getarticle.parsedhtml.getelementsbytagname('p')|select -expand innertext foreach ($i in $innertext){[string]$innertextstring += ($i + "<br>`n")} $a|add-member -notepropertyname "Textonly" -notepropertyvalue $innertextstring $imgs = $getarticle.parsedhtml.body.getelementsbytagname('img') foreach ($i in $imgs){ $src = $nul $alt = $nul $src = $i.src $alt = $i.alt $articleimgs += "<img alt=$alt width=100 src=$i>" } } $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/PowerShell " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit PowerShell ####
#### Reddit Technology #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/technology/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $link = $a.url $comments = $a.num_comments $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl $score = $a.score $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/Technology " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit Technology ####
#### Reddit CompSci #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/compsci/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $link = $a.url $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/CompSci " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $score = $a.score $comments = $a.num_comments if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit Compsci ####
#### Reddit Hardware #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/hardware/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $link = $a.url $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/hardware " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $score = $a.score $comments = $a.num_comments if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit Hardware ####
#### Reddit News #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/news/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $link = $a.url $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/news " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $score = $a.score $comments = $a.num_comments if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit News ####
#### Reddit TechnologyProTips #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/technologyprotips/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $link = $a.url $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/technologyprotips " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $score = $a.score $comments = $a.num_comments if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit TechnologyProTips ####
#### Reddit DailyTechNewsShow #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/dailytechnewsshow/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $link = $a.url $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/dailytechnewsshow " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $score = $a.score $comments = $a.num_comments if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit DailyTechNewsShow ####
#### Reddit tech_news_today #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/tech_news_today/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $link = $a.url $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/tech_news_today " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $score = $a.score $comments = $a.num_comments if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit tech_news_today ####
#### Reddit technews #### $Articles = (Invoke-RestMethod http://www.reddit.com/r/technews/.json).data.children.data $articles = $articles|? {$archive.title -notcontains $_.title} foreach ($a in $articles){ $created = (Get-Date '1/1/1970').Addseconds($a.created) if ($created -lt (get-date).addhours(-24)){continue} $getarticle = $nul $link = $nul $comments = $nul $score = $nul $title = $nul $commenturl = $nul $a|add-member -notepropertyname "date" -notepropertyvalue ([datetime]$created).tooadate() $link = $a.url $a|add-member -notepropertyname "link" -notepropertyvalue $a.url $title = $a.title $a|add-member -notepropertyname "Titlestring" -notepropertyvalue $title $a|add-member -notepropertyname commentURL -notepropertyvalue ("https://www.reddit.com" + $a.permalink) $commenturl = $a.commenturl if ($a.selftext.length -gt 0){$a|add-member -notepropertyname "Textonly" -notepropertyvalue $a.selftext} $a|add-member -notepropertyname "Source" -notepropertyvalue ("Reddit/r/technews " + ($created).tostring()) $a|add-member -notepropertyname "Index" -notepropertyvalue $MasterNumber $index = $a.index $score = $a.score $comments = $a.num_comments if ($a.selftext_html.length -gt 0){$body = $a.selftext_html} if ($a.selftext.length -eq 0){$body = $a.textonly} if ($body.length -gt 0){$body = $body.replace(">",">").replace("<","<").replace("'","'").replace("quot;",'"').replace("&","&").replace("#39;","'").replace("gt;",">").replace("lt;","<")} if ($body.length -eq 0){$a.titlestring = ($a.titlestring + " (link)")} if ($body.length -gt 0){$a.titlestring = ($a.titlestring + " (article)")} $a|add-member -notepropertyname "Article" -notepropertyvalue ("<table width='100%'><tr><td style='font-family: Arial, sans-serif;text-decoration:none;'><span style=color:brown;font-weight:bold;font-size:70%;>($score) Votes <a href=$commenturl>($comments)</a> comments</span><a name=$index href=$link><h3>" + $title + "</h3></a></td></tr></table><table><tr><td style='font-family: Arial, sans-serif;font-size:13;'>" + $body + "</td></tr></table><hr style='border: 2px solid mintcream;'>") if ($a.date -ne $nul){[array]$Thelist += $a} $MasterNumber++ } #### END Reddit technews ####
$TheList = $TheList|sort title -unique #### Only Send New Articles #### #One last check for articles already in the archive foreach ($t in $TheList){ $check = $nul $check = $archive|? {$_.title -eq $t.title} if ($check -eq $nul){[array]$NewOnly += $t} } $TheList = $NewOnly #If less than 15 new articles, then exit the script and wait until the next hour if ($TheList.title.count -lt 15){exit}
if ($TheList.titlestring.count -lt 15){exit}
if ($TheList.article.count -lt 15){exit}
if ($TheList.date.count -lt 15){exit}
#If there's 15 or more articles, I merge the existing archive with the new articles then saves back so there's no article duplication for the next check
$Archive += $TheList|select title,titlestring,source,date
#Since 24 hours is all I grab, I send only 24 hours worth of articles to the archive for future checks
$expiration = (get-date).addhours(-24).tooadate()
$Archive = $Archive|? {[datetime]::FromOADate($_.date) -gt [datetime]::FromOADate($expiration)}
if (test-path \\server\share\scripts){$Archive|select title,titlestring,source,date|export-csv \\server\share\scripts\RSSMailerArchive.csv}
#### New Articles Extracted ####
#### FIND TOP TOPICS #### $IgnoreWords = gc \\server\share\scripts\ignorewords.txt $TitleWordFrequency = ([string]$thelist.titlestring).split(" ") |?{-not [String]::IsNullOrEmpty($_)} | %{[Regex]::Replace($_,'[^a-zA-Z0-9]','')} |group |sort count -Descending $ImportantWords = $TitleWordFrequency |?{$IgnoreWords -notcontains $_.name} | select @{n='ImportanceWeight';e={$_.Count * 0.01}}, @{n='ImportantWord';e={$_.Name}} -First 10 # Adding important word total found in each title foreach ($t in $TheList){ if ($t.wordtotal -eq $nul){$t|add-member -notepropertyname wordtotal -notepropertyvalue 0} $titlestring = $t.titlestring.split(" ") |?{-not [String]::IsNullOrEmpty($_)} | %{[Regex]::Replace($_,'[^a-zA-Z0-9]',' ')} $t.wordtotal = (($titlestring|% {$importantwords.importantword -contains $_}) -ne $false).count } # Sorting by most important word first then titles that are most similar to the first title, then second most important word and titles similar to that selected title. $kw = 1 foreach ($m in $importantwords){ $word = $m.importantword $group = $TheList|? {$_.keyword -eq $nul}|? titlestring -match $word|sort wordtotal -descending $group|add-member -notepropertyname keyword -notepropertyvalue $kw $kw++ $first = $group|select -first 1 foreach ($g in $group){ if ($g.titlestring -eq $first.titlestring){ $subkey = ($first.titlestring.split(' ')).count $g|add-member -notepropertyname subkey -notepropertyvalue $subkey continue } $subkey = 0 $firstts = [array]$first.titlestring.split(' ') $gts = [array]$g.titlestring.split(' ') $sktotal = (($gts|% {$firstts -contains $_}) -ne $false).count $g|add-member -notepropertyname subkey -notepropertyvalue $sktotal } }
#Sorting by keyword then the subkey value 1..10|% { [array]$SortedList += $thelist|? keyword -eq $_|sort subkey -descending } #Add the remaining articles that don't have any of the keywords in their title $SortedList += $TheList|? keyword -eq $nul|sort date -descending #Creating final list for emailing $FinalList = "<center><a name=top></a><table style='width:80%;'>" #Create Title Index. Had to put titles in a table so they could be formatted easier. $odd = 0 foreach ($t in $SortedList){ $index = ("#" + $t.index) $Titlestring = $t.titlestring $name = ("I" + $t.index) if ($odd % 2 -eq 0){$FinalList += ("<tr><td><a name=$name style='line-height: 1.3;background:#f2f2f2;font-family: Arial, Helvetica, sans-serif;text-decoration:none;font-size:90%;font-weight:bold;color:black;' href=$index>$Titlestring</a></td></tr>")} if ($odd % 2 -ne 0){$FinalList += ("<tr><td><a name=$name style='line-height: 1.3;font-family: Arial, Helvetica, sans-serif;text-decoration:none;font-size:90%;font-weight:bold;color:black;' href=$index>$Titlestring</a></td></tr>")} $odd++ } $FinalList += "</table></center>" #Adding the new articles. Adding Top, Article, and Google Search links to each article. foreach ($t in $SortedList){ $name = ("#I" + $t.index) $FinalList += "<a style='font-size:80%;' href=$name>Top</a><br>" $FinalList += ('<span style="font-size:80%">' + $t.Source + '</span><br>') $FinalList += ('<i style="font-size:80%">' + $t.author + '</i><br>') $link = $t.link $FinalList += ("<a style='font-size:80%' href=$link>Article Link</a><br>") $Cleantitle = $t.title -replace ('[^a-zA-Z0-9 ]','') $GoogleTitle = $Cleantitle -replace (' ','+') $FinalList += ("<a style='font-size:80%' href=https://www.google.com/search?q=$GoogleTitle>Google Search</a>") $FinalList += $t.article } #Formatting email title section that presents the top 10 topics $Buzz = (" • Top Topics: " + ([string]$importantwords.importantword).replace(' ',', ')) $ArticleCount = $SortedList.date.count #Emailing behalf of gmail. Only sent to Outlook and iOS email systems. The Top links don't work in iOS but do in Outlook. $SMTPServer = "smtp.gmail.com" $SMTPPort = "587" $Username = "myaccount@gmail.com" $Password = "mypassword" $to = "myworkemail@someserver.com" $subject = ("Daily Tech Summary($ArticleCount) - " + (get-date).tolongdatestring() + $Buzz) $message = New-Object System.Net.Mail.MailMessage $message.subject = $subject $message.body = $FinalList $message.to.add($to) $message.from = $username $message.IsBodyHTML = $true $smtp = New-Object System.Net.Mail.SmtpClient($SMTPServer, $SMTPPort); $smtp.EnableSSL = $true $smtp.Credentials = New-Object System.Net.NetworkCredential($Username, $Password); $smtp.send($message)
No comments:
Post a Comment