File

GNU/Linux PYTHON Script Read Values from JSON File

Another Python script to read values from a JSON file and store them in a mysql database.

import sys
import json
import csv
import MySQLdb
import mysql.connector
from datetime import datetime, date, timedelta

# Get yesterdays date...
yesterday = date.today() - timedelta(1)

# Create database connection...
dbconn = MySQLdb.connect(host='localhost', user='zonpan', passwd='maybe-store-pw-somwhere-else', db='databassename')
cursor = dbconn.cursor()

# Open Mysql Database...
with open('/path/to/directory/yesterday.json') as fd:
     json_data = json.load(fd)

# Insert Query...
add_zopbrengst = ("INSERT INTO ZOpbrengst (Tijdstip, Opbrengst) VALUES (%s, %s)")

# Parse JSON File and insert values...
datum = yesterday.strftime('%Y-%m-%d %H:%M')
opbrengst = json_data['data']['production']['yesterday'].replace(',', '.')
data_zo = (datum, opbrengst)
cursor.execute(add_zopbrengst, data_zo)
print(data_zo)

# Cleanup...
dbconn.commit()
cursor.close()
dbconn.close()

GNU/Linux PYTHON3 Script to Read HUE Sensor DATA

A small python3 script I wrote to read sensor data and store in a logg file.

#!/usr/bin/env python3
import sys
import json
import urllib.request
import requests
import parsedatetime as pdt

# Constants...
cal = pdt.Calendar()
hue_url_sensors = "http://path.to.hue.bridge/api/{api-key}/sensors/"
hue_url_lights = "http://path.to.hue.bridge/api/{api-key}/lights/"
air_url_1 = "http://path.to.other.sensor-1:8082/sensors/"
air_url_2 = "http://path.to.other.sensor-3:8082/sensors/"
rel_url = "http://path.to.relay.raspi:8081/relays/"
light_on = json.dumps({'on': True})
light_off = json.dumps({'on': False})


# Switch function...
def switch(key, value):
  action = 'N/A'
  url = hue_url_lights + key + "/state/"
  if value.lower() == 'off':
    result = requests.put(url, data=light_off.encode('utf-8'))
  elif value.lower() == 'on':
    result = requests.put(url, data=light_on.encode('utf-8'))
  print("INFO: ", "Light=", key, "Action=", value, "Result=", result)

# GetLights function
def getlights():
  # Retrieve huedata (lights)...
  output = ""
  web_data_l = urllib.request.urlopen(hue_url_lights)
  json_data_l = web_data_l.read()
  json_enc_l = web_data_l.info().get_content_charset('utf-8')
  json_lights = json.loads(json_data_l.decode(json_enc_l))
  for key in json_lights:
    if json_lights[key]['state']['reachable'] == True and json_lights[key]['state']['on'] == True:
      state = "on"
    else:
      state = "off"
    output = output + " " + str(key).zfill(2) + "\t" + json_lights[key]['name'].ljust(20) + str(key).ljust(2) + "\t" + state + "\n"
  return output

# GetSensors function
def getSensors():
  # Retreive Sensor Data
  output = ""
  web_data_l = urllib.request.urlopen(hue_url_sensors)
  json_data_l = web_data_l.read()
  json_enc_l = web_data_l.info().get_content_charset('utf-8')
  json_sensors = json.loads(json_data_l.decode(json_enc_l))
  for key in json_sensors:
    isValid = True
    if json_sensors[key]['type'] == 'ZLLLightLevel':
      value = json_sensors[key]['state']['lightlevel']
    elif json_sensors[key]['type'] == 'ZLLPresence':
      value = json_sensors[key]['state']['presence']
    elif json_sensors[key]['type'] == 'ZLLTemperature':
      value = json_sensors[key]['state']['temperature'] /100
    else:
      isValid = False
    if isValid == True:
      output = output + " " + str(key).zfill(2) + "\t"  + str(json_sensors[key]['name']).ljust(20) + "\t\t" + str(value) + ""
  return output

# GetSensors function
def getAirinfo(Url, DeviceId):
  # Retreive Sensor Data
  format = '%Y-%m-%dT%H:%M:%S'
  output = ""
  web_data_l = urllib.request.urlopen(Url)
  json_data_l = web_data_l.read()
  json_enc_l = web_data_l.info().get_content_charset('utf-8')
  json_sensors = json.loads(json_data_l.decode(json_enc_l))['sensors']
  for key in json_sensors:
    output = output + str(key) + ": " + str(json_sensors[key]['value']).ljust(5)
    datetxt = str(cal.parseDT(datetimeString=json_sensors[key]['LastModified'][5:-4])[0].strftime(format))
  return datetxt + " PS" + DeviceId + " " + output + "\n"

# GetRelayState function
def getRelayinfo(Url):
  # Retreive Sensor Data
  r1 = False
  r2 = False
  r8 = False
  VENT = "NA"
  output = ""
  web_data_l = urllib.request.urlopen(Url)
  json_data_l = web_data_l.read()
  json_enc_l = web_data_l.info().get_content_charset('utf-8')
  json_sensors = json.loads(json_data_l.decode(json_enc_l))['relays']
  for key in json_sensors:
    if key == '1':
      if json_sensors[key]['state'] == 1:
        r1=True
    elif key == '2':
      if json_sensors[key]['state'] == 1:
        r2=True
    elif key == '8':
      if json_sensors[key]['state'] == 1:
        r8=True
  if r1 == False and r2 == True:
    VENT = "HIGH"
  if r1 == False and r2 == False:
    VENT = "MEDIUM"
  if r1 == True:
    VENT = "LOW"
  output = " RE\tVENT".ljust(20) + "\t\t" + VENT
  return output


# Construct switch command...
with open("/var/log/ax.log", "a") as logfile:
  logfile.write(getAirinfo(air_url_1, "01"))
  logfile.write(getAirinfo(air_url_2, "02"))

GNU/Linux Move Data Using Rsync

I was doing some maintenance on my local NAS. I used the command below to effectively move data from one location to another without losing file attributes.

rsync -avzhP --remove-source-files /mnt/das-2T-1/source/ /mnt/das-2T-1/destination/ [--dry-run]

You can also temporarily cancel the move and when you start the command again it continues where it stopped.

I also did some internal replication using replication tasks on my TrueNAS device. This creates a snapshot of the data set to replicate to an empty dataset. The destination dataset is overwritten so this option cannot be used to merge datasets. If you want to merge datasets is best to use the rsync option mentioned above.

PowerShell Sftp Retrieve multiple Files (and Remove Source files)

The PowerShell script below retrieves multiple files from a SFTP server source using the native windows SFTP-client in windows After retrieving each individual file it removes the file from the SFTP-server. This specific script makes us of a (preconfigured) ssh-key authentication configuration, but it can easily been rewritten for password authentication.

<#
    Description    : Retrieve (get) and remove files from a sftp source... 
    Keywords       : Sftp, Get and Remove Original, Move

    Majorversion   : 1
    Scriptserver   : LOCALHOST
    Serviceaccount : -

    Author         : Tim van Kooten Niekerk
    Date           : 2023-03-14
#>

<#      CHANGELOG:
        2023-03-14 => Initieel (Tim van Kooten Niekerk)

#>


function fnMoveSftpFiles()
{
  param(
    [string]$sSftpServerHost,
    [string]$sSftpUser,
    [string]$sSftpServerPort = "22",
    [string]$sSftpRemoteFilePath,
    [string]$sSftpRemoteFileMask,
    [string]$sSftpLocalFilePath,
    [string]$sSftpScriptRetrieveClose = "quit"
  )

  # System variables...
  [string]$_sSftpScriptLogFile = $sSftpLocalFilePath + "\get-SftpFiles_" + $(get-date -f yyyy-MM-dd) + '.log'
  [string]$_sSftpScriptRetrieveFiles = "lcd " + $sSftpLocalFilePath + "`ncd " + $sSftpRemoteFilePath + "`n" + "ls -1 " + [string]$sSftpRemoteFileMask
  [string]$_sSftpScriptRetrieveOpen = "lcd " + $sSftpLocalFilePath + "`ncd " + $sSftpRemoteFilePath + "`n"
  [string]$_sSftpScriptRetrieveBody = $_sSftpScriptRetrieveOpen
  [string]$_CurrentTimeStamp = $(get-date -f 'yyyy-MM-dd HH:mm:ss')
  [string]$_InfoDoneWithFiles = $_CurrentTimeStamp + " | INFO | Commands executed: "
  [string]$_InfoDoneWithoutFiles = $_CurrentTimeStamp + " | INFO | No remote files found"
  [string]$_Return = $_CurrentTimeStamp + " | ERROR | Unknown Error"

  # Retrieve file list...
  [array]$aFiles = (write-output $_sSftpScriptRetrieveFiles) | sftp -b - -o Port=$sSftpServerPort $sSftpUser@$sSftpServerHost | select-string -notmatch "sftp>"

  # Create script for retrieving files...
  if ($aFiles.length -gt 0) {
    # Create get and rm script lines for every file in the listing...
    for (($n=0); $n -lt $aFiles.length; $n++) { 
      if ($aFiles[$n].line.length -gt 0)
      { 
        $_sSftpScriptRetrieveBody = $_sSftpScriptRetrieveBody + "get " + $aFiles[$n].Line + "`n"
        $_sSftpScriptRetrieveBody = $_sSftpScriptRetrieveBody + "rm " + $aFiles[$n].Line + "`n"
      }
    }

    # Close script file...
    $_sSftpScriptRetrieveBody = $_sSftpScriptRetrieveBody + $sSftpScriptRetrieveClose

    # Process script...
    [array]$aProcess = (write-output $_sSftpScriptRetrieveBody) | sftp -b - -o Port=$sSftpServerPort $sSftpUser@$sSftpServerHost
  
    # Audit log - Commands executed...
    $_InfoDoneWithFiles + $aProcess | Out-File $_sSftpScriptLogFile -Append
    $_Return = $_InfoDoneWithFiles + $aProcess
    [array]$aProcess = ""
  
  } else {
    # Audit log - No remote files...
    $_InfoDoneWithoutFiles | Out-File $_sSftpScriptLogFile -Append
    $_Return = $_InfoDoneWithoutFiles

  }
  return $_Return
}

PS> fnMoveSftpFiles -sSftpServerHost “example.com” -sSftpUser “user” -sSftpRemoteFilePath “/test” -sSftpRemoteFileMask “*.txt” -sSftpLocalFilePath “D:\Tim\Test” [-sSftpServerPort “22“]

GNU/Linux ASCII Table in CSV/TXT format

Simple ASCII table in (formatted) text and available for download in CSV-format

|   | Binary0  | Oct0 | Dec0 | Hex0 | Chr0 |   | Binary3  | Oct3 | Dec3 | Hex3 | Chr3 |   | Binary6  | Oct6 | Dec6 | Hex6 | Chr6 |
| - | -------- | ---- | ---- | ---- | ---- | - | -------- | ---- | ---- | ---- | ---- | - | -------- | ---- | ---- | ---- | ---- |
|   | 00000000 |  000 |   00 |   00 | NUL  |   | 00110000 |  060 |   48 |   30 | 0    |   | 01100000 |  140 |   96 |   60 | `    |
|   | 00000001 |  001 |   01 |   01 | SOH  |   | 00110001 |  061 |   49 |   31 | 1    |   | 01100001 |  141 |   97 |   61 | a    |
|   | 00000010 |  002 |   02 |   02 | STX  |   | 00110010 |  062 |   50 |   32 | 2    |   | 01100010 |  142 |   98 |   62 | b    |
|   | 00000011 |  003 |   03 |   03 | ETX  |   | 00110011 |  063 |   51 |   33 | 3    |   | 01100011 |  143 |   99 |   63 | c    |
|   | 00000100 |  004 |   04 |   04 | EOT  |   | 00110100 |  064 |   52 |   34 | 4    |   | 01100100 |  144 |  100 |   64 | d    |
|   | 00000101 |  005 |   05 |   05 | ENQ  |   | 00110101 |  065 |   53 |   35 | 5    |   | 01100101 |  145 |  101 |   65 | e    |
|   | 00000110 |  006 |   06 |   06 | ACK  |   | 00110110 |  066 |   54 |   36 | 6    |   | 01100110 |  146 |  102 |   66 | f    |
|   | 00000111 |  007 |   07 |   07 | BEL  |   | 00110111 |  067 |   55 |   37 | 7    |   | 01100111 |  147 |  103 |   67 | g    |
|   | 00001000 |  010 |   08 |   08 | BS   |   | 00111000 |  070 |   56 |   38 | 8    |   | 01101000 |  150 |  104 |   68 | h    |
|   | 00001001 |  011 |   09 |   09 | HT   |   | 00111001 |  071 |   57 |   39 | 9    |   | 01101001 |  151 |  105 |   69 | i    |
|   | 00001010 |  012 |   10 |   0A | LF   |   | 00111010 |  072 |   58 |   3A | :    |   | 01101010 |  152 |  106 |   6A | j    |
|   | 00001011 |  013 |   11 |   0B | VT   |   | 00111011 |  073 |   59 |   3B | ;    |   | 01101011 |  153 |  107 |   6B | k    |
|   | 00001100 |  014 |   12 |   0C | FF   |   | 00111100 |  074 |   60 |   3C | <    |   | 01101100 |  154 |  108 |   6C | l    |
|   | 00001101 |  015 |   13 |   0D | CR   |   | 00111101 |  075 |   61 |   3D | =    |   | 01101101 |  155 |  109 |   6D | m    |
|   | 00001110 |  016 |   14 |   0E | SO   |   | 00111110 |  076 |   62 |   3E | >    |   | 01101110 |  156 |  110 |   6E | n    |
|   | 00001111 |  017 |   15 |   0F | SI   |   | 00111111 |  077 |   63 |   3F | ?    |   | 01101111 |  157 |  111 |   6F | o    |
|   | 00010000 |  020 |   16 |   10 | DLE  |   | 01000000 |  100 |   64 |   40 | @    |   | 01110000 |  160 |  112 |   70 | p    |
|   | 00010001 |  021 |   17 |   11 | DC1  |   | 01000001 |  101 |   65 |   41 | A    |   | 01110001 |  161 |  113 |   71 | q    |
|   | 00010010 |  022 |   18 |   12 | DC2  |   | 01000010 |  102 |   66 |   42 | B    |   | 01110010 |  162 |  114 |   72 | r    |
|   | 00010011 |  023 |   19 |   13 | DC3  |   | 01000011 |  103 |   67 |   43 | C    |   | 01110011 |  163 |  115 |   73 | s    |
|   | 00010100 |  024 |   20 |   14 | DC4  |   | 01000100 |  104 |   68 |   44 | D    |   | 01110100 |  164 |  116 |   74 | t    |
|   | 00010101 |  025 |   21 |   15 | NAK  |   | 01000101 |  105 |   69 |   45 | E    |   | 01110101 |  165 |  117 |   75 | u    |
|   | 00010110 |  026 |   22 |   16 | SYN  |   | 01000110 |  106 |   70 |   46 | F    |   | 01110110 |  166 |  118 |   76 | v    |
|   | 00010111 |  027 |   23 |   17 | ETB  |   | 01000111 |  107 |   71 |   47 | G    |   | 01110111 |  167 |  119 |   77 | w    |
|   | 00011000 |  030 |   24 |   18 | CAN  |   | 01001000 |  110 |   72 |   48 | H    |   | 01111000 |  170 |  120 |   78 | x    |
|   | 00011001 |  031 |   25 |   19 | EM   |   | 01001001 |  111 |   73 |   49 | I    |   | 01111001 |  171 |  121 |   79 | y    |
|   | 00011010 |  032 |   26 |   1A | SUB  |   | 01001010 |  112 |   74 |   4A | J    |   | 01111010 |  172 |  122 |   7A | z    |
|   | 00011011 |  033 |   27 |   1B | ESC  |   | 01001011 |  113 |   75 |   4B | K    |   | 01111011 |  173 |  123 |   7B | {    |
|   | 00011100 |  034 |   28 |   1C | FS   |   | 01001100 |  114 |   76 |   4C | L    |   | 01111100 |  174 |  124 |   7C | |    |
|   | 00011101 |  035 |   29 |   1D | GS   |   | 01001101 |  115 |   77 |   4D | M    |   | 01111101 |  175 |  125 |   7D | }    |
|   | 00011110 |  036 |   30 |   1E | RS   |   | 01001110 |  116 |   78 |   4E | N    |   | 01111110 |  176 |  126 |   7E | ~    |
|   | 00011111 |  037 |   31 |   1F | US   |   | 01001111 |  117 |   79 |   4F | O    |   | 01111111 |  177 |  127 |   7F | DEL  |
|   | 00100000 |  040 |   32 |   20 |      |   | 01010000 |  120 |   80 |   50 | P    |   |          |      |      |      |      |
|   | 00100001 |  041 |   33 |   21 | !    |   | 01010001 |  121 |   81 |   51 | Q    |   |          |      |      |      |      |
|   | 00100010 |  042 |   34 |   22 | "    |   | 01010010 |  122 |   82 |   52 | R    |   |          |      |      |      |      |
|   | 00100011 |  043 |   35 |   23 | #    |   | 01010011 |  123 |   83 |   53 | S    |   |          |      |      |      |      |
|   | 00100100 |  044 |   36 |   24 | $    |   | 01010100 |  124 |   84 |   54 | T    |   |          |      |      |      |      |
|   | 00100101 |  045 |   37 |   25 | %    |   | 01010101 |  125 |   85 |   55 | U    |   |          |      |      |      |      |
|   | 00100110 |  046 |   38 |   26 | &    |   | 01010110 |  126 |   86 |   56 | V    |   |          |      |      |      |      |
|   | 00100111 |  047 |   39 |   27 | '    |   | 01010111 |  127 |   87 |   57 | W    |   |          |      |      |      |      |
|   | 00101000 |  050 |   40 |   28 | (    |   | 01011000 |  130 |   88 |   58 | X    |   |          |      |      |      |      |
|   | 00101001 |  051 |   41 |   29 | )    |   | 01011001 |  131 |   89 |   59 | Y    |   |          |      |      |      |      |
|   | 00101010 |  052 |   42 |   2A | *    |   | 01011010 |  132 |   90 |   5A | Z    |   |          |      |      |      |      |
|   | 00101011 |  053 |   43 |   2B | +    |   | 01011011 |  133 |   91 |   5B | [    |   |          |      |      |      |      |
|   | 00101100 |  054 |   44 |   2C | ,    |   | 01011100 |  134 |   92 |   5C | \    |   |          |      |      |      |      |
|   | 00101101 |  055 |   45 |   2D | -    |   | 01011101 |  135 |   93 |   5D | ]    |   |          |      |      |      |      |
|   | 00101110 |  056 |   46 |   2E | .    |   | 01011110 |  136 |   94 |   5E | ^    |   |          |      |      |      |      |
|   | 00101111 |  057 |   47 |   2F | /    |   | 01011111 |  137 |   95 |   5F | _    |   |          |      |      |      |      |

MSSQL Create Database With Multiple Filegroups (QuickRef)

Query to create a database with multiple filegroups.

CREATE DATABASE [DatabaseName]
ON PRIMARY
  ( Name = 'DBNamePFG', FILENAME = 'D:\DEF\DBNamePFG.mdf', SIZE = 100 MB, FILEGROWTH = 100 MB ),
FILEGROUP [FileGroup2]
  ( Name = 'DBNameFG2a', FILENAME = 'E:\DEF\DBNAmeFG2a.ndf', SIZE = 100 MB, FILEGROWTH = 100 MB ),
  ( Name = 'DBNameFG2b', FILENAME = 'F:\DEF\DBNAmeFG2b.ndf', SIZE = 100 MB, FILEGROWTH = 100 MB )
LOG ON
  ( Name = 'DBNameLog', FILENAME = 'L:\DEF\DBNameLog.ldf', SIZE = 1 GB, FILEGROWTH = 500 MB )