Skip to content

Commit

Permalink
Move quotes database to SQL
Browse files Browse the repository at this point in the history 
Move the quote database from JSON to SQL, for improved data security. With massive JSON files comes an increased chance of file corruption. The SQL database is located on the management portal.
  • Loading branch information
@AnonymousHacker1279
AnonymousHacker1279 committed Dec 28, 2022
1 parent b9aa6e4 commit 0423bd1
Show file tree
Hide file tree
Showing 9 changed files with 401 additions and 373 deletions.
370 changes: 124 additions & 246 deletions Framework/CommandGroups/Quotes.py
View file

Large diffs are not rendered by default.

213 changes: 125 additions & 88 deletions Framework/FileSystemAPI/DataMigration/DataMigrator.py
View file
Original file line number Diff line number Diff line change
@@ -1,93 +1,130 @@
import datetime
import json
import os

from Framework.FileSystemAPI import DatabaseObjects
from Framework.FileSystemAPI.ThreadedLogger import ThreadedLogger
from Framework.GeneralUtilities import GeneralUtilities

logger = None


def initialize(management_portal_handler=None):
global logger

from Framework.FileSystemAPI.ThreadedLogger import ThreadedLogger
logger = ThreadedLogger("DataMigrator", management_portal_handler)


async def migrate_storage_metadata(guild: str, old_version: int) -> None:
metadata_path = os.path.abspath(
os.getcwd() + "/Storage/GuildStorageMetadata.json")

with open(metadata_path, "r") as f:
metadata = json.load(f)

# Migrate from version 1 to version 2
if old_version == 1:
logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 1 to version 2")
old_version = 2
# Version 2 renamed the custom commands metadata file
metadata["guilds"][guild] = 2
object_path = os.path.abspath(os.getcwd() + "/Storage/{}/CustomCommands/metadata.json".format(str(guild)))
os.renames(object_path, object_path.replace("metadata.json", "CommandsMetadata.json"))

with open(metadata_path, "w") as f:
json.dump(metadata, f, indent=4)

# Migrate from version 2 to version 3
if old_version == 2:
logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 2 to version 3")
old_version = 3
# Version 3 changed the author key in the quotes file to use ints instead of strings
metadata["guilds"][guild] = 3

with open(await DatabaseObjects.get_quotes_database(int(guild)), "r") as f:
quotes = json.load(f)

for quote in quotes:
try:
quote["author"] = int(await GeneralUtilities.strip_usernames(str(quote["author"])))
except ValueError:
pass

with open(await DatabaseObjects.get_quotes_database(int(guild)), "w") as f:
json.dump(quotes, f, indent=4)

with open(metadata_path, "w") as f:
json.dump(metadata, f)

# Migrate from version 3 to version 4
if old_version == 3:
logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 3 to version 4")
old_version = 4
# Version 4 added a date field and a "quoted_by" field. Old entries will have the missing fields
# set with a value of "Unknown"
metadata["guilds"][guild] = 4

with open(await DatabaseObjects.get_quotes_database(int(guild)), "r") as f:
quotes = json.load(f)

for quote in quotes:
try:
quote["date"] = "Unknown"
quote["quoted_by"] = "Unknown"
except ValueError:
pass

with open(await DatabaseObjects.get_quotes_database(int(guild)), "w") as f:
json.dump(quotes, f, indent=4)

with open(metadata_path, "w") as f:
json.dump(metadata, f)

# Migrate from version 4 to version 5
if old_version == 4:
logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 4 to version 5")
old_version = 5
# Version 5 removed the Modules.json file
metadata["guilds"][guild] = 5

object_path = os.path.abspath(os.getcwd() + "/Storage/{}/Settings/Modules.json".format(str(guild)))
os.remove(object_path)

with open(metadata_path, "w") as f:
json.dump(metadata, f)
from Framework.ManagementPortal.ManagementPortalHandler import ManagementPortalHandler


class DataMigrator:

def __init__(self, management_portal_handler: ManagementPortalHandler):
self.logger = ThreadedLogger("DataMigrator", management_portal_handler)
self.mp = management_portal_handler

async def migrate_storage_metadata(self, guild: str, old_version: int) -> None:
metadata_path = os.path.abspath(
os.getcwd() + "/Storage/GuildStorageMetadata.json")

with open(metadata_path, "r") as metadata_file:
metadata = json.load(metadata_file)

# Migrate from version 1 to version 2
if old_version == 1:
self.logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 1 to version 2")
old_version = 2
# Version 2 renamed the custom commands metadata file
metadata["guilds"][guild] = 2
object_path = os.path.abspath(os.getcwd() + "/Storage/{}/CustomCommands/metadata.json".format(str(guild)))
os.renames(object_path, object_path.replace("metadata.json", "CommandsMetadata.json"))

# Migrate from version 2 to version 3
if old_version == 2:
self.logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 2 to version 3")
old_version = 3
# Version 3 changed the author key in the quotes file to use ints instead of strings
metadata["guilds"][guild] = 3

with open(await DatabaseObjects.get_quotes_database(int(guild)), "r") as quotes_file:
quotes = json.load(quotes_file)

for quote in quotes:
try:
quote["author"] = int(await GeneralUtilities.strip_usernames(str(quote["author"])))
except ValueError:
pass

with open(await DatabaseObjects.get_quotes_database(int(guild)), "w") as quotes_file:
json.dump(quotes, quotes_file, indent=4)

# Migrate from version 3 to version 4
if old_version == 3:
self.logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 3 to version 4")
old_version = 4
# Version 4 added a date field and a "quoted_by" field. Old entries will have the missing fields
# set with a value of "Unknown"
metadata["guilds"][guild] = 4

with open(await DatabaseObjects.get_quotes_database(int(guild)), "r") as quotes_file:
quotes = json.load(quotes_file)

for quote in quotes:
try:
quote["date"] = "Unknown"
quote["quoted_by"] = "Unknown"
except ValueError:
pass

with open(await DatabaseObjects.get_quotes_database(int(guild)), "w") as quotes_file:
json.dump(quotes, quotes_file, indent=4)

# Migrate from version 4 to version 5
if old_version == 4:
self.logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 4 to version 5")
old_version = 5
# Version 5 removed the Modules.json file
metadata["guilds"][guild] = 5

object_path = os.path.abspath(os.getcwd() + "/Storage/{}/Settings/Modules.json".format(str(guild)))
os.remove(object_path)

# Migrate from version 5 to version 6
if old_version == 5:
self.logger.log_info("Migrating guild storage metadata for guild " + guild + " from version 5 to version 6")
old_version = 6
# Version 6 migrates all JSON storages to a MySQL database hosted on its management portal
metadata["guilds"][guild] = 6

# Migrate the quotes database
with open(await DatabaseObjects.get_quotes_database(int(guild)), "r") as quotes_file:
json_data = json.load(quotes_file)

# Replace all "Unknown" date values with an empty timestamp
for quote in json_data:
if quote["date"] == "Unknown":
quote["date"] = datetime.datetime(1970, 1, 1, 0, 0, 0).isoformat()

# Replace all "Unknown" quoted_by values with an empty ID
for quote in json_data:
if quote["quoted_by"] == "Unknown":
quote["quoted_by"] = 0

# Check the author IDs and ensure it is a valid integer
delete_list = []
for quote in json_data:
try:
quote["author"] = int(quote["author"])
except ValueError:
# The quote will be deleted
# Store deleted quotes in a separate file as a backup
delete_list.append(quote)
json_data.remove(quote)

if len(delete_list) > 0:
# Dump the deleted quotes to a file
backup_file_name = "DeletedQuotesBackup_{}.json".format(guild)
with open(await DatabaseObjects.get_backup_directory() + "\\" + backup_file_name, "w") as backup_file:
json.dump(delete_list, backup_file, indent=4)

self.logger.log_warning("Deleted {} quotes from guild {} due to invalid author IDs".format(len(delete_list), guild))

data = json.dumps(json_data)
await self.mp.data_migration.migrate_quotes(int(guild), data)

# Delete the old quotes database
os.remove(await DatabaseObjects.get_quotes_database(int(guild)))

with open(metadata_path, "w") as metadata_file:
json.dump(metadata, metadata_file, indent=4)
15 changes: 10 additions & 5 deletions Framework/FileSystemAPI/FileAPI.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import os

from Framework.FileSystemAPI import DefaultDatabaseSchemas
from Framework.FileSystemAPI.DataMigration.DataMigrator import migrate_storage_metadata
from Framework.FileSystemAPI.DataMigration.DataMigrator import DataMigrator

logger = None

Expand Down Expand Up @@ -45,7 +45,7 @@ async def create_empty_file(file: str) -> None:
f.write("")


async def check_storage_metadata(current_database_version: int, guilds) -> None:
async def check_storage_metadata(current_database_version: int, data_migrator: DataMigrator, guilds) -> None:
object_path = os.path.abspath(
os.getcwd() + "/Storage/GuildStorageMetadata.json")
directory_path = os.path.dirname(object_path)
Expand All @@ -62,10 +62,15 @@ async def check_storage_metadata(current_database_version: int, guilds) -> None:
try:
if metadata["guilds"][str(guild.id)] < current_database_version:
logger.log_info("Migrating guild storage metadata for guild " + str(guild.id))
await migrate_storage_metadata(str(guild.id), metadata["guilds"][str(guild.id)])
await data_migrator.migrate_storage_metadata(str(guild.id), metadata["guilds"][str(guild.id)])
except KeyError:
logger.log_info("Adding guild storage metadata for new guild " + str(guild.id))

# Reload the metadata file to make sure we don't overwrite any changes
with open(object_path, "r") as f:
metadata = json.load(f)

metadata["guilds"][str(guild.id)] = current_database_version

with open(object_path, "w") as f:
json.dump(metadata, f, indent=4)
with open(object_path, "w") as f:
json.dump(metadata, f, indent=4)
4 changes: 2 additions & 2 deletions Framework/GeneralUtilities/QuoteUtils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ async def prepare_quote(ctx, embed, author, content, quoteID, date, quotedBy) ->
contentExcludingLinks = ""
iteration = 0

if date and quotedBy != "Unknown":
if date != "1970-01-01 00:00:00":
iso_date = datetime.fromisoformat(date)
readable_date = str(iso_date.month) + "/" + str(iso_date.day) + "/" + str(iso_date.year) \
+ " at " + str(iso_date.hour) + \
Expand Down Expand Up @@ -44,7 +44,7 @@ async def prepare_quote(ctx, embed, author, content, quoteID, date, quotedBy) ->
embed.description = author_user
embed.set_footer(text="Added " + readable_date + " by " + quoted_by_user)
else:
embed.description = '> "' + content + '"\n'
embed.description = '> "' + contentExcludingLinks + '"\n'
embed.description += " - " + author_user
embed.set_image(url=links[0])
embed.set_footer(text="Added " + str(readable_date) + " by " + quoted_by_user)
Expand Down
19 changes: 19 additions & 0 deletions Framework/ManagementPortal/APIEndpoints.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
from enum import Enum


class APIEndpoints(str, Enum):
READY = "/v2/bot_ready.php"
UPDATE_LATENCY = "/v2/bot_update_latency.php"
UPDATE_COMMAND_USED = "/v2/bot_update_command_used.php"
LOG_DATA = "/v2/bot_log_data.php"
CHECK_PENDING_COMMANDS = "/v2/bot_check_pending_commands.php"
UPDATE_COMMAND_COMPLETED = "/v2/bot_update_command_completed.php"
GET_CONFIGURATION = "/v2/configurations/portal_get_configuration.php"
MIGRATE_QUOTES = "/v2/migrations/migrate_quotes.php"
GET_QUOTE = "/v2/modules/quotes/get_quote.php"
ADD_QUOTE = "/v2/modules/quotes/add_quote.php"
REMOVE_QUOTE = "/v2/modules/quotes/remove_quote.php"
GET_QUOTE_COUNT = "/v2/modules/quotes/get_quote_count.php"
EDIT_QUOTE = "/v2/modules/quotes/edit_quote.php"
SEARCH_QUOTES = "/v2/modules/quotes/search_quotes.php"
LIST_RECENT_QUOTES = "/v2/modules/quotes/list_recent_quotes.php"
Loading

0 comments on commit 0423bd1

Please sign in to comment.