I started working on a project with my friend Justin in 2022, we were able to put several months in and conducted a lot of experiments.
To learn more about what we have been trying to do: RADENGINEERING.TECH
We haven’t given up on the project, but we both have to make money, to get by, and we don’t have the time to put much more into it …
In order to promote this project, I was asked to build a simple RSS newsreader, operating using JS8 protocol, accessing some of the software’s API and using log files for monitoring. Not a lot of code allowed for a pretty cool experience, where headlines were being sent using CB RADIO, 11M, 27.245 MHZ.
Do with it what you will.
SQL Code (for MySQL): https://planetarystatusreport.com/RSS.TXT
#Daniel John Sullivan, CEO, Rad Engineering, 3/23/22
#This is a news agent/script, a prototype
#Service, utilizing JS8 as a proxy agent
#to encode messages and decode from our
#transceiver
#JS8 Callsign for this service - N3W5 or NEWS ...
#1. monitor directed file for requests
#2. respond to new requests
#3. grab RSS feed data
#there are a few python modules that need
#to be installed: maidenhead, feedparser, bs4
# anyascii, and perhaps one or two others
# just be comfortable using pip3 to install
# these other modules / libraries
from __future__ import print_function
import os
import feedparser
import os.path, time
import json
import math
import time
import maidenhead as mh
import urllib.parse as pr
import xml.etree.ElementTree as ET
from bs4 import BeautifulSoup as BS
from requests import get
from os.path import exists
from socket import socket, AF_INET, SOCK_STREAM
from decimal import Decimal
from datetime import datetime, date, timedelta
from anyascii import anyascii
import mysql.connector
### globals ###
DPath = "/home/pi/.local/share/JS8Call/DIRECTED.TXT"
# server/port info for connecting to the JS8 Call application
# while it is running, it can act as a message server/proxy
# for your radio kit ... we are still figuring out the API
# so right now, it's just 'send' capabilities we are using
# - DJS, 3/19/22
JS8Server = "127.0.0.1"
JS8PortNum = 2442
#make sure you open port 2442 prior to opening JS8 application
#ubuntu command: sudo ufw allow 2442
server = (JS8Server, JS8PortNum)
servern = "localhost";
portn = 3306
usern = "DBUSER";
passw = "DBPASSWORD";
dbn = "DBNAME";
#mode 1: just top stories from each feed, sent just once
#mode 2: active engagement mode
#mode 3: just one service
newsServiceM3 = "ONION"
newsMode = 3
bigSleep = 90
### end of globals ###
def from_message(content):
try:
return json.loads(content)
except ValueError:
return {}
def to_message(typ, value='', params=None):
if params is None:
params = {}
return json.dumps({'type': typ, 'value': value, 'params': params})
class JS8Client(object):
def __init__(self):
self.PttON = False
self.LastPtt = datetime.now()
def process(self, message):
typ = message.get('type', '')
value = message.get('value', '')
params = message.get('params', {})
if not typ:
return
if typ in ('RX.ACTIVITY',):
# skip
return
if value and typ == "RIG.PTT":
if value == "on":
self.PttON = True
self.LastPtt = datetime.now()
print("PTT ON")
if value == "off":
self.PttON = False
print("PTT OFF")
def send(self, *args, **kwargs):
params = kwargs.get('params', {})
if '_ID' not in params:
params['_ID'] = '{}'.format(int(time.time()*1000))
kwargs['params'] = params
message = to_message(*args, **kwargs)
self.sock.send((message + '\n').encode()) # remember to send the newline at the end :)
def GetStationStatus(self):
self.sock = socket(AF_INET, SOCK_STREAM)
self.sock.connect(server)
self.connected = True
try:
self.send("TX.GET_TEXT", "")
content = self.sock.recv(65500)
message = json.loads(content)
typ = message.get('type', '')
value = message.get('value', '')
if typ == "TX.TEXT":
vt = value.strip()
if len(vt) < 1:
return "OPEN"
else:
return "CLOSED"
else:
return "CLOSED"
except:
return "CLOSED"
finally:
self.sock.close()
def SendToJS8(self, JS8Message):
self.sock = socket(AF_INET, SOCK_STREAM)
self.sock.connect(server)
self.connected = True
try:
self.send("TX.SEND_MESSAGE", JS8Message)
content = self.sock.recv(65500)
print(content)
except:
print("Error sending message to JS8 via API.")
finally:
self.sock.close()
def close(self):
self.connected = False
#### END OF CLASS DEFINITION FOR JS8 API INTERFACE ####
def GetArt(number):
# Connect with the MySQL Server
cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
qry = "select ARTICLE, SOURCE, LINK from RSS where ID = %s" % (number)
cur = cnx.cursor(buffered=True)
cur.execute(qry)
retRes = cur.fetchall()
cnx.close()
return retRes[0]
def GetTopHourly(source):
# Connect with the MySQL Server
cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
qry = "select ID, TITLE, PUBLISHED, SOURCE, length(ARTICLE) as LOF from RSS where SOURCE = '%s' order by PUBLISHED desc limit 1" % source
cur = cnx.cursor(buffered=True)
cur.execute(qry)
retRes = cur.fetchall()
cnx.close()
return retRes
def GetTop(source, number):
# Connect with the MySQL Server
cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
qry = "select ID, TITLE, PUBLISHED, SOURCE, length(ARTICLE) as LOF from RSS where SOURCE = '%s' order by PUBLISHED desc limit %s" % (source, number)
cur = cnx.cursor(buffered=True)
cur.execute(qry)
retRes = cur.fetchall()
cnx.close()
return retRes
def AlreadySaved(link):
# Connect with the MySQL Server
cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
qry = "select ID from RSS where LINK = '" + link + "'"
cur = cnx.cursor(buffered=True)
cur.execute(qry)
cur.fetchall()
rc = cur.rowcount
cnx.close()
if rc > 0:
return True
else:
return False
def SaveRSS(source, title, link, published, article):
tit = title.replace("'", "''")
clean_text = anyascii(article)
art = str(clean_text)
art = art.replace("'", "''")
if len(art) > 5000:
art = art[0:5000]
cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
cur = cnx.cursor()
qry = """
INSERT INTO RSS
(SOURCE,
LINK,
TITLE,
PUBLISHED,
ARTICLE)
VALUES
(%s,%s,%s,%s,%s)
"""
val = (source, link, tit, published, art)
cur.execute(qry, val)
cnx.commit()
cnx.close()
def GrabRSS(RssURL, SourceName):
hdrs = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
NewsFeed = feedparser.parse(RssURL)
for na in NewsFeed.entries:
try:
print(na.title)
print(na.link)
print(na.published)
print(na.published_parsed)
except:
continue
if AlreadySaved(na.link):
continue
print("*************************")
response = get(na.link, None, headers=hdrs)
print(na.keys())
soup = BS(response.content, 'html.parser')
txtChunk = ""
for data in soup.find_all("p"):
txtval = data.get_text()
txtval = txtval.strip()
txtarr = txtval.split()
if len(txtarr) == 1:
continue
if "posted" in txtval and ("hours" in txtval or "days" in txtval) and len(txtarr) == 4:
continue
if txtval == "No Search Results Found":
continue
if txtval == "Terms of Service":
continue
if txtval == "Advertise with us":
continue
if txtval == "Media Inquiries":
continue
txtChunk += " " + txtval + "\n"
tyr = na.published_parsed[0]
tmn = na.published_parsed[1]
tdy = na.published_parsed[2]
thr = na.published_parsed[3]
tmi = na.published_parsed[4]
tsc = na.published_parsed[5]
ptms = "%s-%s-%s %s:%s:%s" % (tyr, tmn, tdy, thr, tmi, tsc)
SaveRSS(SourceName, na.title, na.link, ptms, txtChunk.strip())
print(txtChunk.strip())
def debugHere():
input("Press enter to continue ...")
def clearConsole():
command = 'clear'
if os.name in ('nt', 'dos'): # If Machine is running on Windows, use cls
command = 'cls'
os.system(command)
def TopStoriesFrom(news, numof):
s = JS8Client()
ti = GetTop(news, numof)
for t in ti:
pubarr = str(t[2]).split()
pubdte = pubarr[0]
ct = t[1].replace("''","'")
# ID, TITLE, PUBLISHED, SOURCE, LOF
lof = int(t[4])
moreLink = ""
#if lof > 40:
# moreLink = "4MORE: N3WZ [ART]: %s" % t[0]
#headline = "%s HEADLINE [%s] '%s', #%s %s" % ("@ALLCALL", pubdte, ct, t[3], moreLink)
#headline = headline.strip()
headline = "%s HEADLINE [%s] '%s', #%s" % ("@ALLCALL", pubdte, ct, t[3])
print(headline)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(2)
s.SendToJS8(headline)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(2)
time.sleep(bigSleep)
def TopStoriesFromAll():
s = JS8Client()
ti = GetTopHourly("INFOWARS")
tn = GetTopHourly("NYT")
tz = GetTopHourly("ZEROHEDGE")
ty = GetTopHourly("YAHOO")
tc = GetTopHourly("CNN")
tb = GetTopHourly("BBC")
tops = []
tops.append(ti[0])
tops.append(tz[0])
tops.append(ty[0])
tops.append(tc[0])
tops.append(tn[0])
tops.append(tb[0])
for t in tops:
pubarr = str(t[2]).split()
pubdte = pubarr[0]
ct = t[1].replace("''","'")
# ID, TITLE, PUBLISHED, SOURCE, LOF
lof = int(t[4])
moreLink = ""
if lof > 40:
moreLink = "4MORE: N3WZ [ART]: %s" % t[0]
headline = "%s HEADLINE [%s] '%s', #%s %s" % ("@ALLCALL", pubdte, ct, t[3], moreLink)
print(headline)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(2)
s.SendToJS8(headline)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(2)
time.sleep(bigSleep)
def TopStories(messageInfo):
s = JS8Client()
try:
#2S5: N3WZ [TOP]: NYT, 1
mp = messageInfo.split(':')
if len(mp) < 3:
return
callsign = mp[0].strip()
step1 = mp[2].strip()
step2 = step1.split(',')
rn = step2[1].strip().split()
src = step2[0].strip()
numberOf = rn[0].strip()
print("Handling %s request ..." % src)
print(messageInfo)
print("From: " + callsign)
print("Top: " + numberOf)
tops = GetTop(src, numberOf)
for t in tops:
pubarr = str(t[2]).split()
pubdte = pubarr[0]
# ID, TITLE, PUBLISHED
ct = t[1].replace("''","'")
# ID, TITLE, PUBLISHED, SOURCE, LOF
lof = int(t[4])
moreLink = ""
if lof > 40:
moreLink = "4MORE: N3WZ [ART]: %s" % t[0]
headline = "%s HEADLINE [%s] '%s', #%s %s" % (callsign, pubdte, ct, t[3], moreLink)
print(headline)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(5)
s.SendToJS8(headline)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(5)
time.sleep(bigSleep)
except:
print("An exception occurred: getting top stories")
def handleArticle(messageInfo):
try:
#K7IAC: N3W5 [ART]: 237 ♢
s = JS8Client()
mp = messageInfo.split(':')
if len(mp) < 3:
return
callsign = mp[0].strip()
step1 = mp[2].strip().split()
number = step1[0].strip()
print("Handling article request ...")
print(messageInfo)
print("From: " + callsign)
print("Article: " + number)
recd = GetArt(number)
art = recd[0]
src = recd[1]
lnk = recd[2]
if len(art) > 150:
art2 = art.replace("\n", " ")
art2 = art2.replace("''","'")
print("bigger than 150 chars")
if len(art2) > 500:
art2 = art2[0:500]
pts = math.ceil((len(art2)/150))
buffr = ""
part = 1
for c in art2:
buffr += c
if len(buffr) == 150:
artInfo = "%s ART_%s %s #%s (%s/%s)" % (callsign, number, buffr, src, str(part), str(pts))
print(artInfo)
s.SendToJS8(artInfo)
buffr = ""
part += 1
time.sleep(1)
while s.GetStationStatus() == "CLOSED":
print("NEWS SERVICE TIME: " + str(datetime.now()))
time.sleep(1)
time.sleep(60)
if len(buffr) > 0:
artInfo = "%s ART_%s %s #%s (%s/%s)" % (callsign, number, buffr, src, str(part), str(pts))
print(artInfo)
s.SendToJS8(artInfo)
buffr = ""
time.sleep(bigSleep)
else:
art2 = art.replace("\n", " ")
art2 = art2.replace("''","'")
# ARTICLE, SOURCE
artInfo = "%s ART_%s %s ... (MORE @ %s)" % (callsign, number, art2, src)
print(artInfo)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(5)
s.SendToJS8(artInfo)
while s.GetStationStatus() == "CLOSED":
print("Waiting to send ..." + str(datetime.now()))
time.sleep(5)
time.sleep(bigSleep)
except:
print("An exception has occurred: getting news article")
def NewsCycle(DirectedPath):
prevDLineNo = 0
newsl = "news_line.txt"
if(exists(newsl)):
tf = open(newsl, "r")
tfs = tf.read().strip()
if(tfs != ""):
prevDLineNo = int(tfs)
tf.close()
readerf = open(DirectedPath)
CALLSIGN = ""
UTCDTM = ""
MSGINFO = ""
dirLine = 0
uploaded = 0
try:
# Further file processing goes here
for x in readerf:
#0 UTC,1 FREQ,2 OFFSET,3 SNR, 4 MESSAGE
recd = x.split('\t')
UTCDTM = recd[0]
OFFSET = recd[2]
SNR = recd[3]
FREQ = recd[1]
MSG = recd[4]
dirLine += 1
if(dirLine > prevDLineNo):
if "[ART]:" in MSG:
handleArticle(MSG)
if "[TOP]:" in MSG:
TopStories(MSG)
wf = open(newsl, "w")
wf.write(str(dirLine))
wf.close()
finally:
readerf.close()
def CycleFeeds():
infowars = "https://www.infowars.com/rss.xml"
zh = "https://feeds.feedburner.com/zerohedge/feed"
yahoo = "https://news.yahoo.com/rss/"
cnn = "http://rss.cnn.com/rss/cnn_topstories.rss"
bbc = "http://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml"
nyt = "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml"
onion = "https://www.theonion.com/rss"
bb = "https://babylonbee.com/feed"
print("Grabbing Babylon Bee ...")
GrabRSS(bb, "BB")
print("Grabbing ONION ...")
GrabRSS(onion, "ONION")
print("Grabbing INFOWARS ...")
GrabRSS(infowars, "INFOWARS")
print("Grabbing ZEROHEDGE ...")
GrabRSS(zh, "ZEROHEDGE")
print("Grabbing YAHOO ...")
GrabRSS(yahoo, "YAHOO")
print("Grabbing CNN ...")
GrabRSS(cnn, "CNN")
print("Grabbing BBC ...")
GrabRSS(bbc, "BBC")
print("Grabbing NYT ...")
GrabRSS(nyt, "NYT")
# FEEDS:
# 1. INFOWARS: https://www.infowars.com/rss.xml
# 2. ZEROHEDGE: https://feeds.feedburner.com/zerohedge/feed
# 3. YAHOO: https://news.yahoo.com/rss/
# 4. CNN: http://rss.cnn.com/rss/cnn_topstories.rss
clearConsole()
time.sleep(2)
print("Starting NEWS Server .")
time.sleep(1)
clearConsole()
print("Starting NEWS Server ..")
time.sleep(1)
clearConsole()
print("Starting NEWS Server ...")
time.sleep(1)
clearConsole()
print("Starting NEWS Server ....")
time.sleep(1)
clearConsole()
CycleFeeds()
if newsMode == 1:
TopStoriesFromAll()
if newsMode == 2:
ptime = datetime.now()
while True:
ntime = datetime.now()
print("NEWS SERVICE: " + str(ntime))
time.sleep(1)
clearConsole()
NewsCycle(DPath)
clearConsole()
tdiff = ntime - ptime
if tdiff.seconds > (60 * 5):
print("Grabbing RSS feed info ...")
ptime = datetime.now()
CycleFeeds()
if newsMode == 3:
TopStoriesFrom(newsServiceM3, 4)