warn/warn_docstring.go - third_party/github/bazelbuild/buildtools - Git at Google

 /*
 Copyright 2020 Google LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     https://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

 package warn

 import (
 	"fmt"
 	"regexp"
 	"strings"

 	"github.com/bazelbuild/buildtools/build"
 )

 // FunctionLengthDocstringThreshold is a limit for a function size (in statements), above which
 // a public function is required to have a docstring.
 const FunctionLengthDocstringThreshold = 5

 // getDocstring returns a docstring of the statements and true if it exists.
 // Otherwise it returns the first non-comment statement and false.
 func getDocstring(stmts []build.Expr) (*build.Expr, bool) {
 	for i, stmt := range stmts {
 		if stmt == nil {
 			continue
 		}
 		switch stmt.(type) {
 		case *build.CommentBlock:
 			continue
 		case *build.StringExpr:
 			return &stmts[i], true
 		default:
 			return &stmts[i], false
 		}
 	}
 	return nil, false
 }

 func moduleDocstringWarning(f *build.File) []*LinterFinding {
 	if f.Type != build.TypeDefault && f.Type != build.TypeBzl {
 		return nil
 	}
 	if stmt, ok := getDocstring(f.Stmt); stmt != nil && !ok {
 		start, _ := (*stmt).Span()
 		end := build.Position{
 			Line:     start.Line,
 			LineRune: start.LineRune + 1,
 			Byte:     start.Byte + 1,
 		}
 		finding := makeLinterFinding(*stmt, `The file has no module docstring.
 A module docstring is a string literal (not a comment) which should be the first statement of a file (it may follow comment lines).`)
 		finding.End = end
 		return []*LinterFinding{finding}
 	}
 	return nil
 }

 func stmtsCount(stmts []build.Expr) int {
 	result := 0
 	for _, stmt := range stmts {
 		result++
 		switch stmt := stmt.(type) {
 		case *build.IfStmt:
 			result += stmtsCount(stmt.True)
 			result += stmtsCount(stmt.False)
 		case *build.ForStmt:
 			result += stmtsCount(stmt.Body)
 		}
 	}
 	return result
 }

 // docstringInfo contains information about a function docstring
 type docstringInfo struct {
 	hasHeader    bool                      // whether the docstring has a one-line header
 	args         map[string]build.Position // map of documented arguments, the values are line numbers
 	returns      bool                      // whether the return value is documented
 	deprecated   bool                      // whether the function is marked as deprecated
 	argumentsPos build.Position            // line of the `Arguments:` block (not `Args:`), if it exists
 }

 // countLeadingSpaces returns the number of leading spaces of a string.
 func countLeadingSpaces(s string) int {
 	spaces := 0
 	for _, c := range s {
 		if c == ' ' {
 			spaces++
 		} else {
 			break
 		}
 	}
 	return spaces
 }

 var argRegex = regexp.MustCompile(`^ *(\*?\*?\w*)( *\([\w\ ,<>\[\]]+\))?:`)

 // parseFunctionDocstring parses a function docstring and returns a docstringInfo object containing
 // the parsed information about the function, its arguments and its return value.
 func parseFunctionDocstring(doc *build.StringExpr) docstringInfo {
 	start, _ := doc.Span()
 	indent := start.LineRune - 1
 	prefix := strings.Repeat(" ", indent)
 	lines := strings.Split(doc.Value, "\n")

 	// Trim "/r" in the end of the lines to parse CRLF-formatted files correctly
 	for i, line := range lines {
 		lines[i] = strings.TrimRight(line, "\r")
 	}

 	info := docstringInfo{}
 	info.args = make(map[string]build.Position)

 	isArgumentsDescription := false // Whether the currently parsed block is an 'Args:' section
 	argIndentation := 1000000       // Indentation at which previous arg documentation started

 	for i := range lines {
 		lines[i] = strings.TrimRight(lines[i], " ")
 	}

 	// The first non-empty line should be a single-line header
 	for i, line := range lines {
 		if line == "" {
 			continue
 		}
 		if i == len(lines)-1 || lines[i+1] == "" {
 			info.hasHeader = true
 		}
 		break
 	}

 	// Search for Args: and Returns: sections
 	for i, line := range lines {
 		switch line {
 		case prefix + "Arguments:":
 			info.argumentsPos = build.Position{
 				Line:     start.Line + i,
 				LineRune: indent,
 			}
 			isArgumentsDescription = true
 			continue
 		case prefix + "Args:":
 			isArgumentsDescription = true
 			continue
 		case prefix + "Returns:":
 			isArgumentsDescription = false
 			info.returns = true
 			continue
 		case prefix + "Deprecated:":
 			isArgumentsDescription = false
 			info.deprecated = true
 			continue
 		}

 		if isArgumentsDescription {
 			newIndentation := countLeadingSpaces(line)

 			if line != "" && newIndentation <= indent {
 				// The indented block is over
 				isArgumentsDescription = false
 				continue
 			} else if newIndentation > argIndentation {
 				// Continuation of the previous argument description
 				continue
 			} else {
 				// Maybe a new argument is described here
 				result := argRegex.FindStringSubmatch(line)
 				if len(result) > 1 {
 					argIndentation = newIndentation
 					info.args[result[1]] = build.Position{
 						Line:     start.Line + i,
 						LineRune: indent + argIndentation,
 					}
 				}
 			}
 		}
 	}
 	return info
 }

 func hasReturnValues(def *build.DefStmt) bool {
 	result := false
 	build.WalkStatements(def, func(expr build.Expr, stack []build.Expr) (err error) {
 		if _, ok := expr.(*build.DefStmt); ok && len(stack) > 0 {
 			// Don't go into inner function definitions
 			return &build.StopTraversalError{}
 		}

 		ret, ok := expr.(*build.ReturnStmt)
 		if ok && ret.Result != nil {
 			result = true
 		}
 		return
 	})
 	return result
 }

 // isDocstringRequired returns whether a function is required to has a docstring.
 // A docstring is required for public functions if they are long enough (at least 5 statements)
 func isDocstringRequired(def *build.DefStmt) bool {
 	if start, _ := def.Span(); start.LineRune > 1 {
 		// Nested functions don't require docstrings
 		return false
 	}
 	return !strings.HasPrefix(def.Name, "_") && stmtsCount(def.Body) >= FunctionLengthDocstringThreshold
 }

 func functionDocstringWarning(f *build.File) []*LinterFinding {
 	var findings []*LinterFinding

 	// Docstrings are required only for top-level functions
 	for _, stmt := range f.Stmt {
 		def, ok := stmt.(*build.DefStmt)
 		if !ok {
 			continue
 		}

 		if !isDocstringRequired(def) {
 			continue
 		}

 		if _, ok = getDocstring(def.Body); ok {
 			continue
 		}

 		message := fmt.Sprintf(`The function %q has no docstring.
 A docstring is a string literal (not a comment) which should be the first statement of a function body (it may follow comment lines).`, def.Name)
 		finding := makeLinterFinding(def, message)
 		finding.End = def.ColonPos
 		findings = append(findings, finding)
 	}
 	return findings
 }

 func functionDocstringHeaderWarning(f *build.File) []*LinterFinding {
 	var findings []*LinterFinding

 	build.WalkStatements(f, func(expr build.Expr, stack []build.Expr) (err error) {
 		def, ok := expr.(*build.DefStmt)
 		if !ok {
 			return
 		}

 		doc, ok := getDocstring(def.Body)
 		if !ok {
 			return
 		}

 		info := parseFunctionDocstring((*doc).(*build.StringExpr))

 		if !info.hasHeader {
 			message := fmt.Sprintf("The docstring for the function %q should start with a one-line summary.", def.Name)
 			findings = append(findings, makeLinterFinding(*doc, message))
 		}
 		return
 	})
 	return findings
 }

 func functionDocstringArgsWarning(f *build.File) []*LinterFinding {
 	var findings []*LinterFinding

 	build.WalkStatements(f, func(expr build.Expr, stack []build.Expr) (err error) {
 		def, ok := expr.(*build.DefStmt)
 		if !ok {
 			return
 		}

 		doc, ok := getDocstring(def.Body)
 		if !ok {
 			return
 		}

 		info := parseFunctionDocstring((*doc).(*build.StringExpr))

 		if info.argumentsPos.LineRune > 0 {
 			argumentsEnd := info.argumentsPos
 			argumentsEnd.LineRune += len("Arguments:")
 			argumentsEnd.Byte += len("Arguments:")
 			finding := makeLinterFinding(*doc, `Prefer "Args:" to "Arguments:" when documenting function arguments.`)
 			finding.Start = info.argumentsPos
 			finding.End = argumentsEnd
 			findings = append(findings, finding)
 		}

 		if !isDocstringRequired(def) && len(info.args) == 0 {
 			return
 		}

 		// If a docstring is required or there are any arguments described, check for their integrity.

 		// Check whether all arguments are documented.
 		notDocumentedArguments := []string{}
 		paramNames := make(map[string]bool)
 		for _, param := range def.Params {
 			name, op := build.GetParamName(param)
 			if name == "" {
 				continue
 			}
 			name = op + name  // *args or **kwargs
 			paramNames[name] = true
 			if _, ok := info.args[name]; !ok {
 				notDocumentedArguments = append(notDocumentedArguments, name)
 			}
 		}

 		// Check whether all existing arguments are commented
 		if len(notDocumentedArguments) > 0 {
 			message := fmt.Sprintf("Argument %q is not documented.", notDocumentedArguments[0])
 			plural := ""
 			if len(notDocumentedArguments) > 1 {
 				message = fmt.Sprintf(
 					`Arguments "%s" are not documented.`,
 					strings.Join(notDocumentedArguments, `", "`),
 				)
 				plural = "s"
 			}

 			if len(info.args) == 0 {
 				// No arguments are documented maybe the Args: block doesn't exist at all or
 				// formatted improperly. Add extra information to the warning message
 				message += fmt.Sprintf(`

 If the documentation for the argument%s exists but is not recognized by Buildifier
 make sure it follows the line "Args:" which has the same indentation as the opening """,
 and the argument description starts with "<argument_name>:" and indented with at least
 one (preferably two) space more than "Args:", for example:

     def %s(%s):
         """Function description.

         Args:
           %s: argument description, can be
             multiline with additional indentation.
         """`, plural, def.Name, notDocumentedArguments[0], notDocumentedArguments[0])
 			}

 			findings = append(findings, makeLinterFinding(*doc, message))
 		}

 		// Check whether all documented arguments actually exist in the function signature.
 		for name, pos := range info.args {
 			if paramNames[name] {
 				continue
 			}
 			msg := fmt.Sprintf("Argument %q is documented but doesn't exist in the function signature.", name)
 			// *args and **kwargs should be documented with asterisks
 			for _, asterisks := range []string{"*", "**"} {
 				if paramNames[asterisks+name] {
 					msg += fmt.Sprintf(` Do you mean "%s%s"?`, asterisks, name)
 					break
 				}
 			}
 			posEnd := pos
 			posEnd.LineRune += len(name)
 			finding := makeLinterFinding(*doc, msg)
 			finding.Start = pos
 			finding.End = posEnd
 			findings = append(findings, finding)
 		}
 		return
 	})
 	return findings
 }

 func functionDocstringReturnWarning(f *build.File) []*LinterFinding {
 	var findings []*LinterFinding

 	build.WalkStatements(f, func(expr build.Expr, stack []build.Expr) (err error) {
 		def, ok := expr.(*build.DefStmt)
 		if !ok {
 			return
 		}

 		doc, ok := getDocstring(def.Body)
 		if !ok {
 			return
 		}

 		info := parseFunctionDocstring((*doc).(*build.StringExpr))

 		// Check whether the return value is documented
 		if isDocstringRequired(def) && hasReturnValues(def) && !info.returns {
 			message := fmt.Sprintf("Return value of %q is not documented.", def.Name)
 			findings = append(findings, makeLinterFinding(*doc, message))
 		}
 		return
 	})
 	return findings
 }
	/*
	Copyright 2020 Google LLC

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

	https://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
	*/

	package warn

	import (
	"fmt"
	"regexp"
	"strings"

	"github.com/bazelbuild/buildtools/build"
	)

	// FunctionLengthDocstringThreshold is a limit for a function size (in statements), above which
	// a public function is required to have a docstring.
	const FunctionLengthDocstringThreshold = 5

	// getDocstring returns a docstring of the statements and true if it exists.
	// Otherwise it returns the first non-comment statement and false.
	func getDocstring(stmts []build.Expr) (*build.Expr, bool) {
	for i, stmt := range stmts {
	if stmt == nil {
	continue
	}
	switch stmt.(type) {
	case *build.CommentBlock:
	continue
	case *build.StringExpr:
	return &stmts[i], true
	default:
	return &stmts[i], false
	}
	}
	return nil, false
	}

	func moduleDocstringWarning(f build.File) []LinterFinding {
	if f.Type != build.TypeDefault && f.Type != build.TypeBzl {
	return nil
	}
	if stmt, ok := getDocstring(f.Stmt); stmt != nil && !ok {
	start, _ := (*stmt).Span()
	end := build.Position{
	Line: start.Line,
	LineRune: start.LineRune + 1,
	Byte: start.Byte + 1,
	}
	finding := makeLinterFinding(*stmt, `The file has no module docstring.
	A module docstring is a string literal (not a comment) which should be the first statement of a file (it may follow comment lines).`)
	finding.End = end
	return []*LinterFinding{finding}
	}
	return nil
	}

	func stmtsCount(stmts []build.Expr) int {
	result := 0
	for _, stmt := range stmts {
	result++
	switch stmt := stmt.(type) {
	case *build.IfStmt:
	result += stmtsCount(stmt.True)
	result += stmtsCount(stmt.False)
	case *build.ForStmt:
	result += stmtsCount(stmt.Body)
	}
	}
	return result
	}

	// docstringInfo contains information about a function docstring
	type docstringInfo struct {
	hasHeader bool // whether the docstring has a one-line header
	args map[string]build.Position // map of documented arguments, the values are line numbers
	returns bool // whether the return value is documented
	deprecated bool // whether the function is marked as deprecated
	argumentsPos build.Position // line of the `Arguments:` block (not `Args:`), if it exists
	}

	// countLeadingSpaces returns the number of leading spaces of a string.
	func countLeadingSpaces(s string) int {
	spaces := 0
	for _, c := range s {
	if c == ' ' {
	spaces++
	} else {
	break
	}
	}
	return spaces
	}

	var argRegex = regexp.MustCompile(`^ (\?\?\w)( *\([\w\ ,<>\[\]]+\))?:`)

	// parseFunctionDocstring parses a function docstring and returns a docstringInfo object containing
	// the parsed information about the function, its arguments and its return value.
	func parseFunctionDocstring(doc *build.StringExpr) docstringInfo {
	start, _ := doc.Span()
	indent := start.LineRune - 1
	prefix := strings.Repeat(" ", indent)
	lines := strings.Split(doc.Value, "\n")

	// Trim "/r" in the end of the lines to parse CRLF-formatted files correctly
	for i, line := range lines {
	lines[i] = strings.TrimRight(line, "\r")
	}

	info := docstringInfo{}
	info.args = make(map[string]build.Position)

	isArgumentsDescription := false // Whether the currently parsed block is an 'Args:' section
	argIndentation := 1000000 // Indentation at which previous arg documentation started

	for i := range lines {
	lines[i] = strings.TrimRight(lines[i], " ")
	}

	// The first non-empty line should be a single-line header
	for i, line := range lines {
	if line == "" {
	continue
	}
	if i == len(lines)-1 \|\| lines[i+1] == "" {
	info.hasHeader = true
	}
	break
	}

	// Search for Args: and Returns: sections
	for i, line := range lines {
	switch line {
	case prefix + "Arguments:":
	info.argumentsPos = build.Position{
	Line: start.Line + i,
	LineRune: indent,
	}
	isArgumentsDescription = true
	continue
	case prefix + "Args:":
	isArgumentsDescription = true
	continue
	case prefix + "Returns:":
	isArgumentsDescription = false
	info.returns = true
	continue
	case prefix + "Deprecated:":
	isArgumentsDescription = false
	info.deprecated = true
	continue
	}

	if isArgumentsDescription {
	newIndentation := countLeadingSpaces(line)

	if line != "" && newIndentation <= indent {
	// The indented block is over
	isArgumentsDescription = false
	continue
	} else if newIndentation > argIndentation {
	// Continuation of the previous argument description
	continue
	} else {
	// Maybe a new argument is described here
	result := argRegex.FindStringSubmatch(line)
	if len(result) > 1 {
	argIndentation = newIndentation
	info.args[result[1]] = build.Position{
	Line: start.Line + i,
	LineRune: indent + argIndentation,
	}
	}
	}
	}
	}
	return info
	}

	func hasReturnValues(def *build.DefStmt) bool {
	result := false
	build.WalkStatements(def, func(expr build.Expr, stack []build.Expr) (err error) {
	if _, ok := expr.(*build.DefStmt); ok && len(stack) > 0 {
	// Don't go into inner function definitions
	return &build.StopTraversalError{}
	}

	ret, ok := expr.(*build.ReturnStmt)
	if ok && ret.Result != nil {
	result = true
	}
	return
	})
	return result
	}

	// isDocstringRequired returns whether a function is required to has a docstring.
	// A docstring is required for public functions if they are long enough (at least 5 statements)
	func isDocstringRequired(def *build.DefStmt) bool {
	if start, _ := def.Span(); start.LineRune > 1 {
	// Nested functions don't require docstrings
	return false
	}
	return !strings.HasPrefix(def.Name, "_") && stmtsCount(def.Body) >= FunctionLengthDocstringThreshold
	}

	func functionDocstringWarning(f build.File) []LinterFinding {
	var findings []*LinterFinding

	// Docstrings are required only for top-level functions
	for _, stmt := range f.Stmt {
	def, ok := stmt.(*build.DefStmt)
	if !ok {
	continue
	}

	if !isDocstringRequired(def) {
	continue
	}

	if _, ok = getDocstring(def.Body); ok {
	continue
	}

	message := fmt.Sprintf(`The function %q has no docstring.
	A docstring is a string literal (not a comment) which should be the first statement of a function body (it may follow comment lines).`, def.Name)
	finding := makeLinterFinding(def, message)
	finding.End = def.ColonPos
	findings = append(findings, finding)
	}
	return findings
	}

	func functionDocstringHeaderWarning(f build.File) []LinterFinding {
	var findings []*LinterFinding

	build.WalkStatements(f, func(expr build.Expr, stack []build.Expr) (err error) {
	def, ok := expr.(*build.DefStmt)
	if !ok {
	return
	}

	doc, ok := getDocstring(def.Body)
	if !ok {
	return
	}

	info := parseFunctionDocstring((doc).(build.StringExpr))

	if !info.hasHeader {
	message := fmt.Sprintf("The docstring for the function %q should start with a one-line summary.", def.Name)
	findings = append(findings, makeLinterFinding(*doc, message))
	}
	return
	})
	return findings
	}

	func functionDocstringArgsWarning(f build.File) []LinterFinding {
	var findings []*LinterFinding

	build.WalkStatements(f, func(expr build.Expr, stack []build.Expr) (err error) {
	def, ok := expr.(*build.DefStmt)
	if !ok {
	return
	}

	doc, ok := getDocstring(def.Body)
	if !ok {
	return
	}

	info := parseFunctionDocstring((doc).(build.StringExpr))

	if info.argumentsPos.LineRune > 0 {
	argumentsEnd := info.argumentsPos
	argumentsEnd.LineRune += len("Arguments:")
	argumentsEnd.Byte += len("Arguments:")
	finding := makeLinterFinding(*doc, `Prefer "Args:" to "Arguments:" when documenting function arguments.`)
	finding.Start = info.argumentsPos
	finding.End = argumentsEnd
	findings = append(findings, finding)
	}

	if !isDocstringRequired(def) && len(info.args) == 0 {
	return
	}

	// If a docstring is required or there are any arguments described, check for their integrity.

	// Check whether all arguments are documented.
	notDocumentedArguments := []string{}
	paramNames := make(map[string]bool)
	for _, param := range def.Params {
	name, op := build.GetParamName(param)
	if name == "" {
	continue
	}
	name = op + name // args or *kwargs
	paramNames[name] = true
	if _, ok := info.args[name]; !ok {
	notDocumentedArguments = append(notDocumentedArguments, name)
	}
	}

	// Check whether all existing arguments are commented
	if len(notDocumentedArguments) > 0 {
	message := fmt.Sprintf("Argument %q is not documented.", notDocumentedArguments[0])
	plural := ""
	if len(notDocumentedArguments) > 1 {
	message = fmt.Sprintf(
	`Arguments "%s" are not documented.`,
	strings.Join(notDocumentedArguments, `", "`),
	)
	plural = "s"
	}

	if len(info.args) == 0 {
	// No arguments are documented maybe the Args: block doesn't exist at all or
	// formatted improperly. Add extra information to the warning message
	message += fmt.Sprintf(`

	If the documentation for the argument%s exists but is not recognized by Buildifier
	make sure it follows the line "Args:" which has the same indentation as the opening """,
	and the argument description starts with "<argument_name>:" and indented with at least
	one (preferably two) space more than "Args:", for example:

	def %s(%s):
	"""Function description.

	Args:
	%s: argument description, can be
	multiline with additional indentation.
	"""`, plural, def.Name, notDocumentedArguments[0], notDocumentedArguments[0])
	}

	findings = append(findings, makeLinterFinding(*doc, message))
	}

	// Check whether all documented arguments actually exist in the function signature.
	for name, pos := range info.args {
	if paramNames[name] {
	continue
	}
	msg := fmt.Sprintf("Argument %q is documented but doesn't exist in the function signature.", name)
	// args and *kwargs should be documented with asterisks
	for _, asterisks := range []string{"", "*"} {
	if paramNames[asterisks+name] {
	msg += fmt.Sprintf(` Do you mean "%s%s"?`, asterisks, name)
	break
	}
	}
	posEnd := pos
	posEnd.LineRune += len(name)
	finding := makeLinterFinding(*doc, msg)
	finding.Start = pos
	finding.End = posEnd
	findings = append(findings, finding)
	}
	return
	})
	return findings
	}

	func functionDocstringReturnWarning(f build.File) []LinterFinding {
	var findings []*LinterFinding

	build.WalkStatements(f, func(expr build.Expr, stack []build.Expr) (err error) {
	def, ok := expr.(*build.DefStmt)
	if !ok {
	return
	}

	doc, ok := getDocstring(def.Body)
	if !ok {
	return
	}

	info := parseFunctionDocstring((doc).(build.StringExpr))

	// Check whether the return value is documented
	if isDocstringRequired(def) && hasReturnValues(def) && !info.returns {
	message := fmt.Sprintf("Return value of %q is not documented.", def.Name)
	findings = append(findings, makeLinterFinding(*doc, message))
	}
	return
	})
	return findings
	}