Month: June 2023

I started working on a project with my friend Justin in 2022, we were able to put several months in and conducted a lot of experiments.

To learn more about what we have been trying to do: RADENGINEERING.TECH

We haven’t given up on the project, but we both have to make money, to get by, and we don’t have the time to put much more into it …

In order to promote this project, I was asked to build a simple RSS newsreader, operating using JS8 protocol, accessing some of the software’s API and using log files for monitoring. Not a lot of code allowed for a pretty cool experience, where headlines were being sent using CB RADIO, 11M, 27.245 MHZ.

Do with it what you will.

SQL Code (for MySQL): https://planetarystatusreport.com/RSS.TXT

#Daniel John Sullivan, CEO, Rad Engineering, 3/23/22

#This is a news agent/script, a prototype
#Service, utilizing JS8 as a proxy agent
#to encode messages and decode from our
#transceiver

#JS8 Callsign for this service - N3W5 or NEWS ...

#1. monitor directed file for requests
#2. respond to new requests
#3. grab RSS feed data

#there are a few python modules that need
#to be installed: maidenhead, feedparser, bs4
# anyascii, and perhaps one or two others
# just be comfortable using pip3 to install
# these other modules / libraries

from __future__ import print_function

import os
import feedparser
import os.path, time
import json
import math
import time
import maidenhead as mh
import urllib.parse as pr
import xml.etree.ElementTree as ET
from bs4 import BeautifulSoup as BS
from requests import get
from os.path import exists
from socket import socket, AF_INET, SOCK_STREAM
from decimal import Decimal
from datetime import datetime, date, timedelta
from anyascii import anyascii
import mysql.connector

### globals ###

DPath = "/home/pi/.local/share/JS8Call/DIRECTED.TXT"

# server/port info for connecting to the JS8 Call application
# while it is running, it can act as a message server/proxy
# for your radio kit ... we are still figuring out the API
# so right now, it's just 'send' capabilities we are using
# - DJS, 3/19/22

JS8Server = "127.0.0.1"
JS8PortNum = 2442

#make sure you open port 2442 prior to opening JS8 application
#ubuntu command: sudo ufw allow 2442
server = (JS8Server, JS8PortNum)

servern = "localhost";
portn = 3306
usern = "DBUSER";
passw = "DBPASSWORD";
dbn = "DBNAME";

#mode 1: just top stories from each feed, sent just once
#mode 2: active engagement mode
#mode 3: just one service

newsServiceM3 = "ONION"

newsMode = 3

bigSleep = 90

### end of globals ###

def from_message(content):
	try:
		return json.loads(content)
	except ValueError:
		return {}

def to_message(typ, value='', params=None):
	if params is None:
		params = {}
	return json.dumps({'type': typ, 'value': value, 'params': params})

class JS8Client(object):

	def __init__(self):
		self.PttON = False
		self.LastPtt = datetime.now()

	def process(self, message):
		typ = message.get('type', '')
		value = message.get('value', '')
		params = message.get('params', {})
		if not typ:
			return
		if typ in ('RX.ACTIVITY',):
			# skip
			return

		if value and typ == "RIG.PTT":
			if value == "on":
				self.PttON = True
				self.LastPtt = datetime.now()
				print("PTT ON")
			if value == "off":
				self.PttON = False
				print("PTT OFF")
				
	def send(self, *args, **kwargs):
		params = kwargs.get('params', {})
		if '_ID' not in params:
			params['_ID'] = '{}'.format(int(time.time()*1000))
			kwargs['params'] = params
		message = to_message(*args, **kwargs)
		self.sock.send((message + '\n').encode()) # remember to send the newline at the end :)

	def GetStationStatus(self):
		self.sock = socket(AF_INET, SOCK_STREAM)
		self.sock.connect(server)
		self.connected = True
		try:			
			self.send("TX.GET_TEXT", "")
			content = self.sock.recv(65500)
			message = json.loads(content)
			
			typ = message.get('type', '')
			value = message.get('value', '')
			
			if typ == "TX.TEXT":
				vt = value.strip()
				if len(vt) < 1:
					return "OPEN"
				else:
					return "CLOSED"
			else:
				return "CLOSED"
		except:
			return "CLOSED"
		finally:
			self.sock.close()

	def SendToJS8(self, JS8Message):
		self.sock = socket(AF_INET, SOCK_STREAM)
		self.sock.connect(server)
		self.connected = True
		try:			
			self.send("TX.SEND_MESSAGE", JS8Message)
			content = self.sock.recv(65500)
			print(content)
		except:
			print("Error sending message to JS8 via API.")
		finally:
			self.sock.close()

	def close(self):
		self.connected = False

#### END OF CLASS DEFINITION FOR JS8 API INTERFACE ####

def GetArt(number):
	# Connect with the MySQL Server
	cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
	qry = "select ARTICLE, SOURCE, LINK from RSS where ID = %s" % (number)
	cur = cnx.cursor(buffered=True)
	cur.execute(qry)
	retRes = cur.fetchall()
	cnx.close()
	return retRes[0]

def GetTopHourly(source):
	# Connect with the MySQL Server
	cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
	qry = "select ID, TITLE, PUBLISHED, SOURCE, length(ARTICLE) as LOF from RSS where SOURCE = '%s' order by PUBLISHED desc limit 1" % source
	cur = cnx.cursor(buffered=True)
	cur.execute(qry)
	retRes = cur.fetchall()
	cnx.close()
	return retRes

def GetTop(source, number):
	# Connect with the MySQL Server
	cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
	qry = "select ID, TITLE, PUBLISHED, SOURCE, length(ARTICLE) as LOF from RSS where SOURCE = '%s' order by PUBLISHED desc limit %s" % (source, number)
	cur = cnx.cursor(buffered=True)
	cur.execute(qry)
	retRes = cur.fetchall()
	cnx.close()
	return retRes

def AlreadySaved(link):
	# Connect with the MySQL Server
	cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
	qry = "select ID from RSS where LINK = '" + link + "'"
	cur = cnx.cursor(buffered=True)
	cur.execute(qry)
	cur.fetchall()
	rc = cur.rowcount
	cnx.close()
	if rc > 0:
		return True
	else:
		return False

def SaveRSS(source, title, link, published, article):

	tit = title.replace("'", "''")

	clean_text = anyascii(article)
	
	art = str(clean_text)

	art = art.replace("'", "''")
	
	if len(art) > 5000:
		art = art[0:5000]
	
	cnx = mysql.connector.connect(user=usern, database=dbn, password=passw, host=servern, port=portn)
	
	cur = cnx.cursor()

	qry = """
	INSERT INTO RSS
	(SOURCE, 
	LINK, 
	TITLE, 
	PUBLISHED, 
	ARTICLE) 
	VALUES 
	(%s,%s,%s,%s,%s)
	""" 
	
	val = (source, link, tit, published, art)

	cur.execute(qry, val)

	cnx.commit()
	
	cnx.close()

def GrabRSS(RssURL, SourceName):

	hdrs = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}

	NewsFeed = feedparser.parse(RssURL)

	for na in NewsFeed.entries:

		try:
			print(na.title)
			print(na.link)
			print(na.published)
			print(na.published_parsed)
		except:
			continue
			
		if AlreadySaved(na.link):
			continue

		print("*************************")

		response = get(na.link, None, headers=hdrs)

		print(na.keys())

		soup = BS(response.content, 'html.parser')
	
		txtChunk = ""

		for data in soup.find_all("p"):
			txtval = data.get_text()
			txtval = txtval.strip()
			txtarr = txtval.split()
		
			if len(txtarr) == 1:
				continue
		
			if "posted" in txtval and ("hours" in txtval or "days" in txtval) and len(txtarr) == 4:
				continue
			
			if txtval == "No Search Results Found":
				continue
			
			if txtval == "Terms of Service":
				continue
		
			if txtval == "Advertise with us":
				continue
			
			if txtval == "Media Inquiries":
				continue

			txtChunk += " " + txtval + "\n"
	
		tyr = na.published_parsed[0]
		tmn = na.published_parsed[1]
		tdy = na.published_parsed[2]
		thr = na.published_parsed[3]
		tmi = na.published_parsed[4]
		tsc = na.published_parsed[5]
	
		ptms = "%s-%s-%s %s:%s:%s" % (tyr, tmn, tdy, thr, tmi, tsc)	
	
		SaveRSS(SourceName, na.title, na.link, ptms, txtChunk.strip())
	
		print(txtChunk.strip())

def debugHere():
	input("Press enter to continue ...")

def clearConsole():
	command = 'clear'
	if os.name in ('nt', 'dos'):  # If Machine is running on Windows, use cls
		command = 'cls'
	os.system(command)

def TopStoriesFrom(news, numof):
	
	s = JS8Client()

	ti = GetTop(news, numof)
	
	for t in ti:
			
		pubarr = str(t[2]).split()	
		pubdte = pubarr[0]
		ct = t[1].replace("''","'")
		# ID, TITLE, PUBLISHED, SOURCE, LOF
		
		lof = int(t[4])

		moreLink = ""
		
		#if lof > 40:
		#	moreLink = "4MORE: N3WZ [ART]: %s" % t[0]	
		
		#headline = "%s HEADLINE [%s] '%s', #%s %s" % ("@ALLCALL", pubdte, ct, t[3], moreLink)  
		
		#headline = headline.strip()
		
		headline = "%s HEADLINE [%s] '%s', #%s" % ("@ALLCALL", pubdte, ct, t[3]) 
		
		print(headline)
		
		while s.GetStationStatus() == "CLOSED":
			print("Waiting to send ..." + str(datetime.now()))
			time.sleep(2)
		s.SendToJS8(headline)
		while s.GetStationStatus() == "CLOSED":
			print("Waiting to send ..." + str(datetime.now()))
			time.sleep(2)
		time.sleep(bigSleep)

def TopStoriesFromAll():
	
	s = JS8Client()

	ti = GetTopHourly("INFOWARS")
	tn = GetTopHourly("NYT")
	tz = GetTopHourly("ZEROHEDGE")
	ty = GetTopHourly("YAHOO")
	tc = GetTopHourly("CNN")
	tb = GetTopHourly("BBC")
	
	tops = []
	
	tops.append(ti[0])
	tops.append(tz[0])
	tops.append(ty[0])
	tops.append(tc[0])
	tops.append(tn[0])
	tops.append(tb[0])

	for t in tops:
		pubarr = str(t[2]).split()	
		pubdte = pubarr[0]
		ct = t[1].replace("''","'")
		
		# ID, TITLE, PUBLISHED, SOURCE, LOF
		
		lof = int(t[4])

		moreLink = ""
		
		if lof > 40:
			moreLink = "4MORE: N3WZ [ART]: %s" % t[0]	
		
		headline = "%s HEADLINE [%s] '%s', #%s %s" % ("@ALLCALL", pubdte, ct, t[3], moreLink)  
		
		print(headline)
		while s.GetStationStatus() == "CLOSED":
			print("Waiting to send ..." + str(datetime.now()))
			time.sleep(2)
		s.SendToJS8(headline)
		while s.GetStationStatus() == "CLOSED":
			print("Waiting to send ..." + str(datetime.now()))
			time.sleep(2)
		time.sleep(bigSleep)

def TopStories(messageInfo):
	
	s = JS8Client()
	
	try:	
		#2S5: N3WZ  [TOP]: NYT, 1
		mp = messageInfo.split(':')
		
		if len(mp) < 3:
			return

		callsign = mp[0].strip()
		step1 = mp[2].strip()
		step2 = step1.split(',')
		rn = step2[1].strip().split()
		src = step2[0].strip()	
		numberOf = rn[0].strip()

		print("Handling %s request ..." % src)
		print(messageInfo)
		print("From: " + callsign)
		print("Top: " + numberOf)

		tops = GetTop(src, numberOf)
			
		for t in tops:
			pubarr = str(t[2]).split()	
			pubdte = pubarr[0]
			# ID, TITLE, PUBLISHED
			ct = t[1].replace("''","'")
			
			# ID, TITLE, PUBLISHED, SOURCE, LOF
			
			lof = int(t[4])

			moreLink = ""
			
			if lof > 40:
				moreLink = "4MORE: N3WZ [ART]: %s" % t[0]	
			
			headline = "%s HEADLINE [%s] '%s', #%s %s" % (callsign, pubdte, ct, t[3], moreLink)  
			
			print(headline)
			while s.GetStationStatus() == "CLOSED":
				print("Waiting to send ..." + str(datetime.now()))
				time.sleep(5)
			s.SendToJS8(headline)
			while s.GetStationStatus() == "CLOSED":
				print("Waiting to send ..." + str(datetime.now()))
				time.sleep(5)
			time.sleep(bigSleep)		

	except:	
		print("An exception occurred: getting top stories")
	
def handleArticle(messageInfo):
	
	try:
		#K7IAC: N3W5 [ART]: 237 ♢  
		
		s = JS8Client()
		
		mp = messageInfo.split(':')
		
		if len(mp) < 3:
			return
		
		callsign = mp[0].strip()
		
		step1 = mp[2].strip().split()

		number = step1[0].strip()

		print("Handling article request ...")
		print(messageInfo)
		print("From: " + callsign)
		print("Article: " + number)

		recd = GetArt(number)
		
		art = recd[0]
		src = recd[1]
		lnk = recd[2]
		
		if len(art) > 150:
		
			art2 = art.replace("\n", " ")
			art2 = art2.replace("''","'")
		
			print("bigger than 150 chars")
			
			if len(art2) > 500:
				art2 = art2[0:500]
			
			pts = math.ceil((len(art2)/150))
			buffr = ""
			part = 1
			for c in art2:
				buffr += c
				if len(buffr) == 150:
				
					artInfo = "%s ART_%s %s #%s (%s/%s)" % (callsign, number, buffr, src, str(part), str(pts))  
				
					print(artInfo)
					
					s.SendToJS8(artInfo)
					
					buffr = ""
					part += 1
					time.sleep(1)
					while s.GetStationStatus() == "CLOSED":
						print("NEWS SERVICE TIME: " + str(datetime.now()))
						time.sleep(1)
					time.sleep(60)
			
			if len(buffr) > 0:
				artInfo = "%s ART_%s %s #%s (%s/%s)" % (callsign, number, buffr, src, str(part), str(pts))  
				print(artInfo)
				s.SendToJS8(artInfo)
				buffr = ""
			time.sleep(bigSleep)

		else:		
			art2 = art.replace("\n", " ")
			art2 = art2.replace("''","'")
			# ARTICLE, SOURCE
			artInfo = "%s ART_%s %s ... (MORE @ %s)" % (callsign, number, art2, src)  
			print(artInfo)
			while s.GetStationStatus() == "CLOSED":
				print("Waiting to send ..." + str(datetime.now()))
				time.sleep(5)
			s.SendToJS8(artInfo)
			while s.GetStationStatus() == "CLOSED":
				print("Waiting to send ..." + str(datetime.now()))
				time.sleep(5)
			time.sleep(bigSleep)
	except:
		print("An exception has occurred: getting news article")

def NewsCycle(DirectedPath):
	
	prevDLineNo = 0

	newsl = "news_line.txt"

	if(exists(newsl)):
		tf = open(newsl, "r")
		tfs = tf.read().strip()
		if(tfs != ""):
			prevDLineNo = int(tfs)
		tf.close()

	readerf = open(DirectedPath)

	CALLSIGN = ""
		
	UTCDTM = ""
		
	MSGINFO = ""

	dirLine = 0
		
	uploaded = 0
		
	try:
		# Further file processing goes here
		for x in readerf:

			#0 UTC,1 FREQ,2 OFFSET,3 SNR, 4 MESSAGE

			recd = x.split('\t')

			UTCDTM = recd[0]
			
			OFFSET = recd[2]
			
			SNR = recd[3]
			
			FREQ = recd[1]
			
			MSG = recd[4]
										
			dirLine += 1
				
			if(dirLine > prevDLineNo):
			
				if "[ART]:" in MSG:
					handleArticle(MSG)

				if "[TOP]:" in MSG:
					TopStories(MSG)

				wf = open(newsl, "w")
				wf.write(str(dirLine))
				wf.close()
	
	finally:
		readerf.close()

def CycleFeeds():
	infowars = "https://www.infowars.com/rss.xml"
	zh = "https://feeds.feedburner.com/zerohedge/feed"
	yahoo = "https://news.yahoo.com/rss/"
	cnn = "http://rss.cnn.com/rss/cnn_topstories.rss"
	bbc = "http://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml"
	nyt = "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml"
	onion = "https://www.theonion.com/rss"
	bb = "https://babylonbee.com/feed"
	print("Grabbing Babylon Bee ...")
	GrabRSS(bb, "BB")
	print("Grabbing ONION ...")
	GrabRSS(onion, "ONION")
	print("Grabbing INFOWARS ...")
	GrabRSS(infowars, "INFOWARS")
	print("Grabbing ZEROHEDGE ...")
	GrabRSS(zh, "ZEROHEDGE")
	print("Grabbing YAHOO ...")
	GrabRSS(yahoo, "YAHOO")
	print("Grabbing CNN ...")
	GrabRSS(cnn, "CNN")
	print("Grabbing BBC ...")
	GrabRSS(bbc, "BBC")
	print("Grabbing NYT ...")
	GrabRSS(nyt, "NYT")

# FEEDS:
# 1. INFOWARS: https://www.infowars.com/rss.xml
# 2. ZEROHEDGE: https://feeds.feedburner.com/zerohedge/feed
# 3. YAHOO: https://news.yahoo.com/rss/
# 4. CNN: http://rss.cnn.com/rss/cnn_topstories.rss

clearConsole()
time.sleep(2)
print("Starting NEWS Server .")
time.sleep(1)
clearConsole()
print("Starting NEWS Server ..")
time.sleep(1)
clearConsole()
print("Starting NEWS Server ...")
time.sleep(1)
clearConsole()
print("Starting NEWS Server ....")
time.sleep(1)
clearConsole()

CycleFeeds()

if newsMode == 1:
	TopStoriesFromAll()

if newsMode == 2:

	ptime = datetime.now()
	
	while True:
		ntime = datetime.now()

		print("NEWS SERVICE: " + str(ntime))

		time.sleep(1)
		clearConsole()
		NewsCycle(DPath)	
		clearConsole()

		tdiff = ntime - ptime

		if tdiff.seconds > (60 * 5):
			print("Grabbing RSS feed info ...")
			ptime = datetime.now()
			CycleFeeds()
	
if newsMode == 3:
	TopStoriesFrom(newsServiceM3, 4)