#!/usr/bin/python

#

# Copyright 2014 Google Inc. All Rights Reserved.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""Create documentation for generate API surfaces.

Command-line tool that creates documentation for all APIs listed in discovery.

The documentation is generated from a combination of the discovery document and

the generated API surface itself.

"""

from __future__ import print_function

__author__ = "jcgregorio@google.com (Joe Gregorio)"

import argparse

import collections

import json

import pathlib

import re

import string

import sys

import uritemplate

from googleapiclient.discovery import DISCOVERY_URI, build_from_document

from googleapiclient.http import build_http

DISCOVERY_DOC_DIR = (

pathlib.Path(__file__).resolve().parent

/ "googleapiclient"

/ "discovery_cache"

/ "documents"

)

CSS = """<style>

body, h1, h2, h3, div, span, p, pre, a {

margin: 0;

padding: 0;

border: 0;

font-weight: inherit;

font-style: inherit;

font-size: 100%;

font-family: inherit;

vertical-align: baseline;

}

body {

font-size: 13px;

padding: 1em;

}

h1 {

font-size: 26px;

margin-bottom: 1em;

}

h2 {

font-size: 24px;

margin-bottom: 1em;

}

h3 {

font-size: 20px;

margin-bottom: 1em;

margin-top: 1em;

}

pre, code {

line-height: 1.5;

font-family: Monaco, 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', 'Lucida Console', monospace;

}

pre {

margin-top: 0.5em;

}

h1, h2, h3, p {

font-family: Arial, sans serif;

}

h1, h2, h3 {

border-bottom: solid #CCC 1px;

}

.toc_element {

margin-top: 0.5em;

}

.firstline {

margin-left: 2 em;

}

.method {

margin-top: 1em;

border: solid 1px #CCC;

padding: 1em;

background: #EEE;

}

.details {

font-weight: bold;

font-size: 14px;

}

</style>

"""

METHOD_TEMPLATE = """<div class="method">

<code class="details" id="$name">$name($params)</code>

<pre>$doc</pre>

</div>

"""

COLLECTION_LINK = """<p class="toc_element">

<code><a href="$href">$name()</a></code>

</p>

<p class="firstline">Returns the $name Resource.</p>

"""

METHOD_LINK = """<p class="toc_element">

<code><a href="#$name">$name($params)</a></code></p>

<p class="firstline">$firstline</p>"""

BASE = pathlib.Path(__file__).resolve().parent / "docs" / "dyn"

# Obtain the discovery index and artifacts from googleapis/discovery-artifact-manager

DIRECTORY_URI = "https://raw.githubusercontent.com/googleapis/discovery-artifact-manager/master/discoveries/index.json"

DISCOVERY_URI_TEMPLATE = "https://raw.githubusercontent.com/googleapis/discovery-artifact-manager/master/discoveries/{api}.{apiVersion}.json"

parser = argparse.ArgumentParser(description=__doc__)

parser.add_argument(

"--discovery_uri_template",

default=DISCOVERY_URI_TEMPLATE,

help="URI Template for discovery.",

)

parser.add_argument(

"--discovery_uri",

default="",

help=(

"URI of discovery document. If supplied then only "

"this API will be documented."

),

)

parser.add_argument(

"--directory_uri",

default=DIRECTORY_URI,

help=("URI of directory document. Unused if --discovery_uri" " is supplied."),

)

parser.add_argument(

"--dest", default=BASE, help="Directory name to write documents into."

)

def safe_version(version):

"""Create a safe version of the verion string.

Needed so that we can distinguish between versions

and sub-collections in URIs. I.e. we don't want

adsense_v1.1 to refer to the '1' collection in the v1

version of the adsense api.

Args:

version: string, The version string.

Returns:

The string with '.' replaced with '_'.

"""

return version.replace(".", "_")

def unsafe_version(version):

"""Undoes what safe_version() does.

See safe_version() for the details.

Args:

version: string, The safe version string.

Returns:

The string with '_' replaced with '.'.

"""

return version.replace("_", ".")

def method_params(doc):

"""Document the parameters of a method.

Args:

doc: string, The method's docstring.

Returns:

The method signature as a string.

"""

doclines = doc.splitlines()

if "Args:" in doclines:

begin = doclines.index("Args:")

if "Returns:" in doclines[begin + 1 :]:

end = doclines.index("Returns:", begin)

args = doclines[begin + 1 : end]

else:

args = doclines[begin + 1 :]

parameters = []

sorted_parameters = []

pname = None

desc = ""

def add_param(pname, desc):

if pname is None:

return

if "(required)" not in desc:

pname = pname + "=None"

parameters.append(pname)

else:

# required params should be put straight into sorted_parameters

# to maintain order for positional args

sorted_parameters.append(pname)

for line in args:

m = re.search(r"^\s+([a-zA-Z0-9_]+): (.*)", line)

if m is None:

desc += line

continue

add_param(pname, desc)

pname = m.group(1)

desc = m.group(2)

add_param(pname, desc)

sorted_parameters.extend(sorted(parameters))

sorted_parameters = ", ".join(sorted_parameters)

else:

sorted_parameters = ""

return sorted_parameters

def method(name, doc):

"""Documents an individual method.

Args:

name: string, Name of the method.

doc: string, The methods docstring.

"""

import html

params = method_params(doc)

doc = html.escape(doc)

return string.Template(METHOD_TEMPLATE).substitute(

name=name, params=params, doc=doc

)

def breadcrumbs(path, root_discovery):

"""Create the breadcrumb trail to this page of documentation.

Args:

path: string, Dot separated name of the resource.

root_discovery: Deserialized discovery document.

Returns:

HTML with links to each of the parent resources of this resource.

"""

parts = path.split(".")

crumbs = []

accumulated = []

for i, p in enumerate(parts):

prefix = ".".join(accumulated)

# The first time through prefix will be [], so we avoid adding in a

# superfluous '.' to prefix.

if prefix:

prefix += "."

display = p

if i == 0:

display = root_discovery.get("title", display)

crumbs.append('<a href="{}.html">{}</a>'.format(prefix + p, display))

accumulated.append(p)

return " . ".join(crumbs)

def document_collection(resource, path, root_discovery, discovery, css=CSS):

"""Document a single collection in an API.

Args:

resource: Collection or service being documented.

path: string, Dot separated name of the resource.

root_discovery: Deserialized discovery document.

discovery: Deserialized discovery document, but just the portion that

describes the resource.

css: string, The CSS to include in the generated file.

"""

collections = []

methods = []

resource_name = path.split(".")[-2]

html = [

"<html><body>",

css,

"<h1>%s</h1>" % breadcrumbs(path[:-1], root_discovery),

"<h2>Instance Methods</h2>",

]

# Which methods are for collections.

for name in dir(resource):

if not name.startswith("_") and callable(getattr(resource, name)):

if hasattr(getattr(resource, name), "__is_resource__"):

collections.append(name)

else:

methods.append(name)

# TOC

if collections:

for name in collections:

if not name.startswith("_") and callable(getattr(resource, name)):

href = path + name + ".html"

html.append(

string.Template(COLLECTION_LINK).substitute(href=href, name=name)

)

if methods:

for name in methods:

if not name.startswith("_") and callable(getattr(resource, name)):

doc = getattr(resource, name).__doc__

params = method_params(doc)

firstline = doc.splitlines()[0]

html.append(

string.Template(METHOD_LINK).substitute(

name=name, params=params, firstline=firstline

)

)

if methods:

html.append("<h3>Method Details</h3>")

for name in methods:

dname = name.rsplit("_")[0]

html.append(method(name, getattr(resource, name).__doc__))

html.append("</body></html>")

return "\n".join(html)

def document_collection_recursive(

resource,

path,

root_discovery,

discovery,

doc_destination_dir,

artifact_destination_dir=DISCOVERY_DOC_DIR,

):

html = document_collection(resource, path, root_discovery, discovery)

f = open(pathlib.Path(doc_destination_dir).joinpath(path + "html"), "w")

f.write(html)

f.close()

for name in dir(resource):

if (

not name.startswith("_")

and callable(getattr(resource, name))

and hasattr(getattr(resource, name), "__is_resource__")

and discovery != {}

):

dname = name.rsplit("_")[0]

collection = getattr(resource, name)()

document_collection_recursive(

collection,

path + name + ".",

root_discovery,

discovery["resources"].get(dname, {}),

doc_destination_dir,

artifact_destination_dir,

)

def document_api(

name,

version,

uri,

doc_destination_dir,

artifact_destination_dir=DISCOVERY_DOC_DIR,

):

"""Document the given API.

Args:

name (str): Name of the API.

version (str): Version of the API.

uri (str): URI of the API's discovery document

doc_destination_dir (str): relative path where the reference

documentation should be saved.

artifact_destination_dir (Optional[str]): relative path where the discovery

artifacts should be saved.

"""

http = build_http()

resp, content = http.request(uri)

if resp.status == 200:

discovery = json.loads(content)

service = build_from_document(discovery)

doc_name = "{}.{}.json".format(name, version)

discovery_file_path = artifact_destination_dir / doc_name

revision = None

pathlib.Path(discovery_file_path).touch(exist_ok=True)

# Write discovery artifact to disk if revision equal or newer

with open(discovery_file_path, "r+") as f:

try:

json_data = json.load(f)

revision = json_data["revision"]

except json.JSONDecodeError:

revision = None

if revision is None or discovery["revision"] >= revision:

# Reset position to the beginning

f.seek(0)

# Write the changes to disk

json.dump(discovery, f, indent=0, sort_keys=True)

# Truncate anything left as it's not needed

f.truncate()

elif resp.status == 404:

print(

"Warning: {} {} not found. HTTP Code: {}".format(name, version, resp.status)

)

return

else:

print(

"Warning: {} {} could not be built. HTTP Code: {}".format(

name, version, resp.status

)

)

return

document_collection_recursive(

service,

"{}_{}.".format(name, safe_version(version)),

discovery,

discovery,

doc_destination_dir,

artifact_destination_dir,

)

def document_api_from_discovery_document(

discovery_url, doc_destination_dir, artifact_destination_dir=DISCOVERY_DOC_DIR

):

"""Document the given API.

Args:

discovery_url (str): URI of discovery document.

doc_destination_dir (str): relative path where the reference

documentation should be saved.

artifact_destination_dir (str): relative path where the discovery

artifacts should be saved.

"""

http = build_http()

response, content = http.request(discovery_url)

discovery = json.loads(content)

service = build_from_document(discovery)

name = discovery["version"]

version = safe_version(discovery["version"])

document_collection_recursive(

service,

"{}_{}.".format(name, version),

discovery,

discovery,

doc_destination_dir,

artifact_destination_dir,

)

def generate_all_api_documents(

directory_uri=DIRECTORY_URI,

doc_destination_dir=BASE,

artifact_destination_dir=DISCOVERY_DOC_DIR,

discovery_uri_template=DISCOVERY_URI_TEMPLATE,

):

"""Retrieve discovery artifacts and fetch reference documentations

for all apis listed in the public discovery directory.

args:

directory_uri (Optional[str]): uri of the public discovery directory.

doc_destination_dir (Optional[str]): relative path where the reference

documentation should be saved.

artifact_destination_dir (Optional[str]): relative path where the discovery

artifacts should be saved.

discovery_uri_template (Optional[str]): URI template of the API's discovery

document.

"""

api_directory = collections.defaultdict(list)

http = build_http()

resp, content = http.request(directory_uri)

if resp.status == 200:

directory = json.loads(content)["items"]

for api in directory:

uri = uritemplate.expand(

discovery_uri_template or api["discoveryRestUrl"],

{"api": api["name"], "apiVersion": api["version"]},

)

document_api(

api["name"],

api["version"],

uri,

doc_destination_dir,

artifact_destination_dir,

)

api_directory[api["name"]].append(api["version"])

# sort by api name and version number

for api in api_directory:

api_directory[api] = sorted(api_directory[api])

api_directory = collections.OrderedDict(

sorted(api_directory.items(), key=lambda x: x[0])

)

markdown = []

for api, versions in api_directory.items():

markdown.append("## %s" % api)

for version in versions:

markdown.append(

"* [%s](http://googleapis.github.io/google-api-python-client/docs/dyn/%s_%s.html)"

% (version, api, safe_version(version))

)

markdown.append("\n")

with open(doc_destination_dir / "index.md", "w") as f:

markdown = "\n".join(markdown)

f.write(markdown)

else:

sys.exit("Failed to load the discovery document.")

if __name__ == "__main__":

FLAGS = parser.parse_args(sys.argv[1:])

if FLAGS.discovery_uri:

document_api_from_discovery_document(

discovery_url=FLAGS.discovery_uri,

doc_destination_dir=FLAGS.dest,

)

else:

generate_all_api_documents(

directory_uri=FLAGS.directory_uri,

doc_destination_dir=FLAGS.dest,

discovery_uri_template=FLAGS.discovery_uri_template,

)