#!/usr/bin/env python3

import glob

import json

import os

import argparse

ARG_TYPES = {

"string": "ARG_TYPE_STRING",

"integer": "ARG_TYPE_INTEGER",

"double": "ARG_TYPE_DOUBLE",

"key": "ARG_TYPE_KEY",

"pattern": "ARG_TYPE_PATTERN",

"unix-time": "ARG_TYPE_UNIX_TIME",

"pure-token": "ARG_TYPE_PURE_TOKEN",

"oneof": "ARG_TYPE_ONEOF",

"block": "ARG_TYPE_BLOCK",

}

GROUPS = {

"generic": "COMMAND_GROUP_GENERIC",

"string": "COMMAND_GROUP_STRING",

"list": "COMMAND_GROUP_LIST",

"set": "COMMAND_GROUP_SET",

"sorted_set": "COMMAND_GROUP_SORTED_SET",

"hash": "COMMAND_GROUP_HASH",

"pubsub": "COMMAND_GROUP_PUBSUB",

"transactions": "COMMAND_GROUP_TRANSACTIONS",

"connection": "COMMAND_GROUP_CONNECTION",

"server": "COMMAND_GROUP_SERVER",

"scripting": "COMMAND_GROUP_SCRIPTING",

"hyperloglog": "COMMAND_GROUP_HYPERLOGLOG",

"cluster": "COMMAND_GROUP_CLUSTER",

"sentinel": "COMMAND_GROUP_SENTINEL",

"geo": "COMMAND_GROUP_GEO",

"stream": "COMMAND_GROUP_STREAM",

"bitmap": "COMMAND_GROUP_BITMAP",

"array": "COMMAND_GROUP_ARRAY",

"rate_limit": "COMMAND_GROUP_RATE_LIMIT",

}

def get_optional_desc_string(desc, field, force_uppercase=False):

v = desc.get(field, None)

if v and force_uppercase:

v = v.upper()

ret = "\"%s\"" % v if v else "NULL"

return ret.replace("\n", "\\n")

def check_command_args_key_specs(args, command_key_specs_index_set, command_arg_key_specs_index_set):

if not args:

return True

for arg in args:

if arg.key_spec_index is not None:

assert isinstance(arg.key_spec_index, int)

if arg.key_spec_index not in command_key_specs_index_set:

print("command: %s arg: %s key_spec_index error" % (command.fullname(), arg.name))

return False

command_arg_key_specs_index_set.add(arg.key_spec_index)

if not check_command_args_key_specs(arg.subargs, command_key_specs_index_set, command_arg_key_specs_index_set):

return False

return True

def check_command_key_specs(command):

if not command.key_specs:

return True

assert isinstance(command.key_specs, list)

for cmd_key_spec in command.key_specs:

if "flags" not in cmd_key_spec:

print("command: %s key_specs missing flags" % command.fullname())

return False

if "NOT_KEY" in cmd_key_spec["flags"]:

# Like SUNSUBSCRIBE / SPUBLISH / SSUBSCRIBE

return True

command_key_specs_index_set = set(range(len(command.key_specs)))

command_arg_key_specs_index_set = set()

# Collect key_spec used for each arg, including arg.subarg

if not check_command_args_key_specs(command.args, command_key_specs_index_set, command_arg_key_specs_index_set):

return False

# Check if we have key_specs not used

if command_key_specs_index_set != command_arg_key_specs_index_set:

print("command: %s may have unused key_spec" % command.fullname())

return False

return True

# Globals

subcommands = {} # container_name -> dict(subcommand_name -> Subcommand) - Only subcommands

commands = {} # command_name -> Command - Only commands

class KeySpec(object):

def __init__(self, spec):

self.spec = spec

def struct_code(self):

def _flags_code():

s = ""

for flag in self.spec.get("flags", []):

s += "CMD_KEY_%s|" % flag

return s[:-1] if s else 0

def _begin_search_code():

if self.spec["begin_search"].get("index"):

return "KSPEC_BS_INDEX,.bs.index={%d}" % (

self.spec["begin_search"]["index"]["pos"]

)

elif self.spec["begin_search"].get("keyword"):

return "KSPEC_BS_KEYWORD,.bs.keyword={\"%s\",%d}" % (

self.spec["begin_search"]["keyword"]["keyword"],

self.spec["begin_search"]["keyword"]["startfrom"],

)

elif "unknown" in self.spec["begin_search"]:

return "KSPEC_BS_UNKNOWN,{{0}}"

else:

print("Invalid begin_search! value=%s" % self.spec["begin_search"])

exit(1)

def _find_keys_code():

if self.spec["find_keys"].get("range"):

return "KSPEC_FK_RANGE,.fk.range={%d,%d,%d}" % (

self.spec["find_keys"]["range"]["lastkey"],

self.spec["find_keys"]["range"]["step"],

self.spec["find_keys"]["range"]["limit"]

)

elif self.spec["find_keys"].get("keynum"):

return "KSPEC_FK_KEYNUM,.fk.keynum={%d,%d,%d}" % (

self.spec["find_keys"]["keynum"]["keynumidx"],

self.spec["find_keys"]["keynum"]["firstkey"],

self.spec["find_keys"]["keynum"]["step"]

)

elif "unknown" in self.spec["find_keys"]:

return "KSPEC_FK_UNKNOWN,{{0}}"

else:

print("Invalid find_keys! value=%s" % self.spec["find_keys"])

exit(1)

return "%s,%s,%s,%s" % (

get_optional_desc_string(self.spec, "notes"),

_flags_code(),

_begin_search_code(),

_find_keys_code()

)

def verify_no_dup_names(container_fullname, args):

name_list = [arg.name for arg in args]

name_set = set(name_list)

if len(name_list) != len(name_set):

print("{}: Dup argument names: {}".format(container_fullname, name_list))

exit(1)

class Argument(object):

def __init__(self, parent_name, desc):

self.parent_name = parent_name

self.desc = desc

self.name = self.desc["name"].lower()

if "_" in self.name:

print("{}: name ({}) should not contain underscores".format(self.fullname(), self.name))

exit(1)

self.type = self.desc["type"]

self.key_spec_index = self.desc.get("key_spec_index", None)

self.subargs = []

if self.type in ["oneof", "block"]:

self.display = None

for subdesc in self.desc["arguments"]:

self.subargs.append(Argument(self.fullname(), subdesc))

if len(self.subargs) < 2:

print("{}: oneof or block arg contains less than two subargs".format(self.fullname()))

exit(1)

verify_no_dup_names(self.fullname(), self.subargs)

else:

self.display = self.desc.get("display")

def fullname(self):

return ("%s %s" % (self.parent_name, self.name)).replace("-", "_")

def struct_name(self):

return "%s_Arg" % (self.fullname().replace(" ", "_"))

def subarg_table_name(self):

assert self.subargs

return "%s_Subargs" % (self.fullname().replace(" ", "_"))

def struct_code(self):

"""

Output example:

MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=GETEX_expiration_Subargs

"""

def _flags_code():

s = ""

if self.desc.get("optional", False):

s += "CMD_ARG_OPTIONAL|"

if self.desc.get("multiple", False):

s += "CMD_ARG_MULTIPLE|"

if self.desc.get("multiple_token", False):

assert self.desc.get("multiple", False) # Sanity

s += "CMD_ARG_MULTIPLE_TOKEN|"

return s[:-1] if s else "CMD_ARG_NONE"

s = "MAKE_ARG(\"%s\",%s,%d,%s,%s,%s,%s,%d,%s)" % (

self.name,

ARG_TYPES[self.type],

self.desc.get("key_spec_index", -1),

get_optional_desc_string(self.desc, "token", force_uppercase=True),

get_optional_desc_string(self.desc, "summary"),

get_optional_desc_string(self.desc, "since"),

_flags_code(),

len(self.subargs),

get_optional_desc_string(self.desc, "deprecated_since"),

)

if "display" in self.desc:

s += ",.display_text=\"%s\"" % self.desc["display"].lower()

if self.subargs:

s += ",.subargs=%s" % self.subarg_table_name()

return s

def write_internal_structs(self, f):

if self.subargs:

for subarg in self.subargs:

subarg.write_internal_structs(f)

f.write("/* %s argument table */\n" % self.fullname())

f.write("struct COMMAND_ARG %s[] = {\n" % self.subarg_table_name())

for subarg in self.subargs:

f.write("{%s},\n" % subarg.struct_code())

f.write("};\n\n")

def to_c_name(str):

return str.replace(":", "").replace(".", "_").replace("$", "_")\

.replace("^", "_").replace("*", "_").replace("-", "_") \

.replace("\\", "_").replace("+", "_")

class ReplySchema(object):

def __init__(self, name, desc):

self.name = to_c_name(name)

self.schema = {}

if desc.get("type") == "object":

if desc.get("properties") and desc.get("additionalProperties") is None:

print("%s: Any object that has properties should have the additionalProperties field" % self.name)

exit(1)

elif desc.get("type") == "array":

if desc.get("items") and isinstance(desc["items"], list) and any([desc.get(k) is None for k in ["minItems", "maxItems"]]):

print("%s: Any array that has items should have the minItems and maxItems fields" % self.name)

exit(1)

for k, v in desc.items():

if isinstance(v, dict):

self.schema[k] = ReplySchema("%s_%s" % (self.name, k), v)

elif isinstance(v, list):

self.schema[k] = []

for i, subdesc in enumerate(v):

self.schema[k].append(ReplySchema("%s_%s_%i" % (self.name, k,i), subdesc))

else:

self.schema[k] = v

def write(self, f):

def struct_code(name, k, v):

if isinstance(v, ReplySchema):

t = "JSON_TYPE_OBJECT"

vstr = ".value.object=&%s" % name

elif isinstance(v, list):

t = "JSON_TYPE_ARRAY"

vstr = ".value.array={.objects=%s,.length=%d}" % (name, len(v))

elif isinstance(v, bool):

t = "JSON_TYPE_BOOLEAN"

vstr = ".value.boolean=%d" % int(v)

elif isinstance(v, str):

t = "JSON_TYPE_STRING"

vstr = ".value.string=\"%s\"" % v

elif isinstance(v, int):

t = "JSON_TYPE_INTEGER"

vstr = ".value.integer=%d" % v

return "%s,%s,%s" % (t, json.dumps(k), vstr)

for k, v in self.schema.items():

if isinstance(v, ReplySchema):

v.write(f)

elif isinstance(v, list):

for i, schema in enumerate(v):

schema.write(f)

name = to_c_name("%s_%s" % (self.name, k))

f.write("/* %s array reply schema */\n" % name)

f.write("struct jsonObject *%s[] = {\n" % name)

for i, schema in enumerate(v):

f.write("&%s,\n" % schema.name)

f.write("};\n\n")

f.write("/* %s reply schema */\n" % self.name)

f.write("struct jsonObjectElement %s_elements[] = {\n" % self.name)

for k, v in self.schema.items():

name = to_c_name("%s_%s" % (self.name, k))

f.write("{%s},\n" % struct_code(name, k, v))

f.write("};\n\n")

f.write("struct jsonObject %s = {%s_elements,.length=%d};\n\n" % (self.name, self.name, len(self.schema)))

class Command(object):

def __init__(self, name, desc):

self.name = name.upper()

self.desc = desc

self.group = self.desc["group"]

self.key_specs = self.desc.get("key_specs", [])

self.subcommands = []

self.args = []

for arg_desc in self.desc.get("arguments", []):

self.args.append(Argument(self.fullname(), arg_desc))

verify_no_dup_names(self.fullname(), self.args)

self.reply_schema = None

if "reply_schema" in self.desc:

self.reply_schema = ReplySchema(self.reply_schema_name(), self.desc["reply_schema"])

def fullname(self):

return self.name.replace("-", "_").replace(":", "")

def return_types_table_name(self):

return "%s_ReturnInfo" % self.fullname().replace(" ", "_")

def subcommand_table_name(self):

assert self.subcommands

return "%s_Subcommands" % self.name

def history_table_name(self):

return "%s_History" % (self.fullname().replace(" ", "_"))

def tips_table_name(self):

return "%s_Tips" % (self.fullname().replace(" ", "_"))

def arg_table_name(self):

return "%s_Args" % (self.fullname().replace(" ", "_"))

def key_specs_table_name(self):

return "%s_Keyspecs" % (self.fullname().replace(" ", "_"))

def reply_schema_name(self):

return "%s_ReplySchema" % (self.fullname().replace(" ", "_"))

def struct_name(self):

return "%s_Command" % (self.fullname().replace(" ", "_"))

def history_code(self):

if not self.desc.get("history"):

return ""

s = ""

for tupl in self.desc["history"]:

s += "{\"%s\",\"%s\"},\n" % (tupl[0], tupl[1])

return s

def num_history(self):

if not self.desc.get("history"):

return 0

return len(self.desc["history"])

def tips_code(self):

if not self.desc.get("command_tips"):

return ""

s = ""

for hint in self.desc["command_tips"]:

s += "\"%s\",\n" % hint.lower()

return s

def num_tips(self):

if not self.desc.get("command_tips"):

return 0

return len(self.desc["command_tips"])

def key_specs_code(self):

s = ""

for spec in self.key_specs:

s += "{%s}," % KeySpec(spec).struct_code()

return s[:-1]

def struct_code(self):

"""

Output example:

MAKE_CMD("set","Set the string value of a key","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SET_History,4,SET_Tips,0,setCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_STRING,SET_Keyspecs,1,setGetKeys,5),.args=SET_Args

"""

def _flags_code():

s = ""

for flag in self.desc.get("command_flags", []):

s += "CMD_%s|" % flag

return s[:-1] if s else 0

def _acl_categories_code():

s = ""

for cat in self.desc.get("acl_categories", []):

s += "ACL_CATEGORY_%s|" % cat

return s[:-1] if s else 0

def _doc_flags_code():

s = ""

for flag in self.desc.get("doc_flags", []):

s += "CMD_DOC_%s|" % flag

return s[:-1] if s else "CMD_DOC_NONE"

s = "MAKE_CMD(\"%s\",%s,%s,%s,%s,%s,%s,%s,%s,%s,%d,%s,%d,%s,%d,%s,%s,%s,%d,%s,%d)," % (

self.name.lower(),

get_optional_desc_string(self.desc, "summary"),

get_optional_desc_string(self.desc, "complexity"),

get_optional_desc_string(self.desc, "since"),

_doc_flags_code(),

get_optional_desc_string(self.desc, "replaced_by"),

get_optional_desc_string(self.desc, "deprecated_since"),

"\"%s\"" % self.group,

GROUPS[self.group],

self.history_table_name(),

self.num_history(),

self.tips_table_name(),

self.num_tips(),

self.desc.get("function", "NULL"),

self.desc["arity"],

_flags_code(),

_acl_categories_code(),

self.key_specs_table_name(),

len(self.key_specs),

self.desc.get("get_keys_function", "NULL"),

len(self.args),

)

if self.subcommands:

s += ".subcommands=%s," % self.subcommand_table_name()

if self.args:

s += ".args=%s," % self.arg_table_name()

if self.reply_schema and args.with_reply_schema:

s += ".reply_schema=&%s," % self.reply_schema_name()

return s[:-1]

def write_internal_structs(self, f):

if self.subcommands:

subcommand_list = sorted(self.subcommands, key=lambda cmd: cmd.name)

for subcommand in subcommand_list:

subcommand.write_internal_structs(f)

f.write("/* %s command table */\n" % self.fullname())

f.write("struct COMMAND_STRUCT %s[] = {\n" % self.subcommand_table_name())

for subcommand in subcommand_list:

f.write("{%s},\n" % subcommand.struct_code())

f.write("{0}\n")

f.write("};\n\n")

f.write("/********** %s ********************/\n\n" % self.fullname())

f.write("#ifndef SKIP_CMD_HISTORY_TABLE\n")

f.write("/* %s history */\n" % self.fullname())

code = self.history_code()

if code:

f.write("commandHistory %s[] = {\n" % self.history_table_name())

f.write("%s" % code)

f.write("};\n")

else:

f.write("#define %s NULL\n" % self.history_table_name())

f.write("#endif\n\n")

f.write("#ifndef SKIP_CMD_TIPS_TABLE\n")

f.write("/* %s tips */\n" % self.fullname())

code = self.tips_code()

if code:

f.write("const char *%s[] = {\n" % self.tips_table_name())

f.write("%s" % code)

f.write("};\n")

else:

f.write("#define %s NULL\n" % self.tips_table_name())

f.write("#endif\n\n")

f.write("#ifndef SKIP_CMD_KEY_SPECS_TABLE\n")

f.write("/* %s key specs */\n" % self.fullname())

code = self.key_specs_code()

if code:

f.write("keySpec %s[%d] = {\n" % (self.key_specs_table_name(), len(self.key_specs)))

f.write("%s\n" % code)

f.write("};\n")

else:

f.write("#define %s NULL\n" % self.key_specs_table_name())

f.write("#endif\n\n")

if self.args:

for arg in self.args:

arg.write_internal_structs(f)

f.write("/* %s argument table */\n" % self.fullname())

f.write("struct COMMAND_ARG %s[] = {\n" % self.arg_table_name())

for arg in self.args:

f.write("{%s},\n" % arg.struct_code())

f.write("};\n\n")

if self.reply_schema and args.with_reply_schema:

self.reply_schema.write(f)

class Subcommand(Command):

def __init__(self, name, desc):

self.container_name = desc["container"].upper()

super(Subcommand, self).__init__(name, desc)

def fullname(self):

return "%s %s" % (self.container_name, self.name.replace("-", "_").replace(":", ""))

def create_command(name, desc):

flags = desc.get("command_flags")

if flags and "EXPERIMENTAL" in flags:

print("Command %s is experimental, skipping..." % name)

return

if desc.get("container"):

cmd = Subcommand(name.upper(), desc)

subcommands.setdefault(desc["container"].upper(), {})[name] = cmd

else:

cmd = Command(name.upper(), desc)

commands[name.upper()] = cmd

# MAIN

# Figure out where the sources are

srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src")

parser = argparse.ArgumentParser()

parser.add_argument('--with-reply-schema', action='store_true')

args = parser.parse_args()

# Create all command objects

print("Processing json files...")

for filename in glob.glob('%s/commands/*.json' % srcdir):

with open(filename, "r") as f:

try:

d = json.load(f)

for name, desc in d.items():

create_command(name, desc)

except json.decoder.JSONDecodeError as err:

print("Error processing %s: %s" % (filename, err))

exit(1)

# Link subcommands to containers

print("Linking container command to subcommands...")

for command in commands.values():

assert command.group

if command.name not in subcommands:

continue

for subcommand in subcommands[command.name].values():

assert not subcommand.group or subcommand.group == command.group

subcommand.group = command.group

command.subcommands.append(subcommand)

check_command_error_counter = 0 # An error counter is used to count errors in command checking.

print("Checking all commands...")

for command in commands.values():

if not check_command_key_specs(command):

check_command_error_counter += 1

if check_command_error_counter != 0:

print("Error: There are errors in the commands check, please check the above logs.")

exit(1)

commands_filename = "commands_with_reply_schema" if args.with_reply_schema else "commands"

print("Generating %s.def..." % commands_filename)

with open("%s/%s.def" % (srcdir, commands_filename), "w") as f:

f.write("/* Automatically generated by %s, do not edit. */\n\n" % os.path.basename(__file__))

f.write(

"""

/* We have fabulous commands from

* the fantastic

* Redis Command Table! */

/* Must match redisCommandGroup */

const char *COMMAND_GROUP_STR[] = {

"generic",

"string",

"list",

"set",

"sorted-set",

"hash",

"pubsub",

"transactions",

"connection",

"server",

"scripting",

"hyperloglog",

"cluster",

"sentinel",

"geo",

"stream",

"bitmap",

"array",

"module",

#ifdef ENABLE_GCRA

"rate_limit"

#endif

};

const char *commandGroupStr(int index) {

return COMMAND_GROUP_STR[index];

}

"""

)

command_list = sorted(commands.values(), key=lambda cmd: (cmd.group, cmd.name))

for command in command_list:

command.write_internal_structs(f)

f.write("/* Main command table */\n")

f.write("struct COMMAND_STRUCT redisCommandTable[] = {\n")

curr_group = None

for command in command_list:

if curr_group != command.group:

curr_group = command.group

f.write("/* %s */\n" % curr_group)

f.write("{%s},\n" % command.struct_code())

f.write("{0}\n")

f.write("};\n")

print("All done, exiting.")