Merge branch 'main' of github.com:LMFDB/lmfdb into cmdline

Author: roed314

.github/workflows/matrix_includes.json

@@ -30,12 +30,12 @@
         "server": "devmirror"
     },
     {
-        "files": "lmfdb/tests/test_dynamic_knowls.py lmfdb/tests/test_root.py lmfdb/hecke_algebras/test_hecke_algebras.py lmfdb/tests/test_homepage.py lmfdb/tests/test_random_redirects.py lmfdb/elliptic_curves/test_ell_curves.py lmfdb/elliptic_curves/test_browse_page.py",
+        "files": "lmfdb/tests/test_dynamic_knowls.py lmfdb/tests/test_root.py lmfdb/tests/test_code_coverage.py lmfdb/hecke_algebras/test_hecke_algebras.py lmfdb/tests/test_homepage.py lmfdb/tests/test_random_redirects.py lmfdb/elliptic_curves/test_ell_curves.py lmfdb/elliptic_curves/test_browse_page.py",
         "folders": "elliptic_curves hecke_algebras tests",
         "server": "proddb"
     },
     {
-        "files": "lmfdb/tests/test_dynamic_knowls.py lmfdb/tests/test_root.py lmfdb/hecke_algebras/test_hecke_algebras.py lmfdb/tests/test_homepage.py lmfdb/tests/test_random_redirects.py lmfdb/elliptic_curves/test_ell_curves.py lmfdb/elliptic_curves/test_browse_page.py",
+        "files": "lmfdb/tests/test_dynamic_knowls.py lmfdb/tests/test_root.py lmfdb/tests/test_code_coverage.py lmfdb/hecke_algebras/test_hecke_algebras.py lmfdb/tests/test_homepage.py lmfdb/tests/test_random_redirects.py lmfdb/elliptic_curves/test_ell_curves.py lmfdb/elliptic_curves/test_browse_page.py",
         "folders": "elliptic_curves hecke_algebras tests",
         "server": "devmirror"
     },

.github/workflows/python-package.yml

@@ -90,7 +90,7 @@ jobs:
     - name: checking that we didn't miss any test files
       shell: bash -l {0}
       # If this fails you need to update the file list above and file count
-      run: test $(find lmfdb -name 'test_*.py' -or -name '*_test.py' | wc -l) -eq 41
+      run: test $(find lmfdb -name 'test_*.py' -or -name '*_test.py' | wc -l) -eq 42
 
     - name: Config LMFDB to run tests against proddb
       if: matrix.files != 'lint' && matrix.server == 'proddb'

.github/workflows/snippet_generate.yml

@@ -2,6 +2,8 @@ name: Generate Snippet Code Files
 
 on:
   push:
+    branches: 
+      - main
     paths:
       - '**/code*.yaml'
 
@@ -26,5 +28,5 @@ jobs:
         with:
           commit-message: snippet code test files generated by action
           title: Code snippets from snippet-generate action
-          body: This is an auto-generated PR with code snippets from yaml files
+          body: This is an auto-generated PR with code snippets from yaml files. 
           labels: code snippets, automated pr

Development.md

@@ -39,8 +39,7 @@ Structural Conventions
 
 Below, we describe several of the structural conventions used in the LMFDB. Note
 that we also have styling conventions for the content of individual pages. These
-styling conventions are described in
-[StyleGuide.md](https://github.com/LMFDB/lmfdb/blob/main/StyleGuide.md).
+styling conventions are described in [StyleGuide.md](./StyleGuide.md).
 
 Pages in the LMFDB
 ------------------
@@ -206,11 +205,11 @@ should be
 Here and throughout this document, we say "should" instead of "must", with
 the understanding that some compromises will be inevitable.
 
-Here are some principles:
+Here are some conventions the LMFDB uses:
 
 1. If X is the URL of an object, then L/X is the URL of its standard
    L-function. According to the Langlands program, other L-functions
-   associated to X have the form L(s, X, rho). Examples include
+   associated to X may be written on the the form L(s, X, rho). Examples include
 
    - L(s, f, sym^2)
    - L(s, f, spin)
@@ -261,8 +260,7 @@ Here are some principles:
    should be considered permanent:
 
    - `/ArtinRepresentation/\<dim>/\<conductor>/\<label>`
-   - `/Character/Dirichlet/\<modulus>/\<number>`
-   - `/Character/Hecke/\<number_field>/\<modulus>/\<number>`
+   - `/Character/Dirichlet/\<modulus>/\<label>`
    - `/EllipticCurve/Q/\<label>`
    - `/GaloisGroup/\<label>`
    - `/padicField/\<label>`
@@ -337,7 +335,7 @@ Code Attribution
 ----------------
 
 Each file should begin with a short copyright information, mentioning the people
-who are mainly involved in coding this particular python file. In practice, 
+who are mainly involved in coding this particular python file. 
 
 
 Testing
@@ -356,106 +354,111 @@ Testing
 
 - A code coverage diagnostic can be obtained via
   ```
-  ./test.sh html
+  ./test.sh coverage
   ```
-  it produces beautiful coverage scores in `lmfdb/cover/index.html`
+  which produces coverage scores saved in `lmfdb/htmlcov`.
 
-Pro Tip: Debugging
--------------------
+Code Snippets 
+-------------
 
-Just add
+Many of the LMFDB pages include code snippets which describe how to generate the objects on the page using computer algebra systems such as SageMath, Magma, pari/GP or Oscar. To add code snippets to a new page, the relevant code is placed in a file named `code.yaml`. The file should always define the languages used using the `prompt` tag, for example: 
 ```
-  import pdb; pdb.set_trace()
+prompt:
+  sage:   'sage'
+  pari:   'gp'
+  magma:  'magma'
+  oscar:  'oscar'
 ```
-somewhere (e.g. protected inside a sensible if) this magic
-line and you will end up inside the interactive python
-debugger. there, you can check for the local variables with dir()
-you can execute python code (e.g. to introspect objects)
-and use "pp <var name>" to pretty print variables and
-to continue executing code use the "n" command.
-When you get lost, the command "bt" shows you exactly where you
-are and "up" helps you to get on step up on the stack.
-Of course, "help `<command>`" will tell you more...
-
-Git Tips
-=========
-
-global .gitignore
------------------
 
-Please configure Git to have a global .gitignore for all your projects.
-It should contain all the files which are not project specific, but happen
-on your machine. E.g. temporary files ending in `...~` or `.DS_store`.
-
-[copy paste ready instructions on github](https://help.github.com/articles/ignoring-files#global-gitignore)
-
-.gitconfig
-----------
-
-In your home directory, in the file ~/.gitconfig
-
-```
-[alias]
-        st=status
-        aliases=!git config --get-regexp 'alias.*' | colrm 1 6 | sed 's/[ ]/ = /'
-        ci=commit
-        br=branch
-        co=checkout
-        df=diff
-        who=shortlog -s --
-        ll = log --oneline --graph --decorate -25
-        lla = log --oneline --graph --decorate --all -25
-        wdiff=diff --word-diff=color
-[color]
-    ui = auto
-    branch = auto
-    diff = auto
-    interactive = auto
-    status = auto
+The code snippets should be formatted as template strings, which will then be formatted in the code when the page is loaded.
+```yaml # From number_fields/code.yaml
+field:
+  comment: Define the number field
+  sage: x = polygen(QQ);  K.<a> = NumberField(%s)
+  pari: K = bnfinit(%s, 1)
+  magma: R<x> := PolynomialRing(Rationals()); K<a> := NumberField(%s);
+  oscar: Qx, x = polynomial_ring(QQ); K, a = number_field(%s)
 ```
 
-List-table should always be like
---------------------------------
+### Snippet testing
+
+Sometimes computer algebra systems make breaking changes which render the code snippets invalid. To catch this, there's a testing system for snippets implemented in `lmfdb/tests/generate_snippet_tests.py`. This runs the code line by line in the various CAS-es, excluding magma. It does not test correctness of the results, only consistency, by comparing with previous results stored in the `lmfdb/tests/snippet_tests` directory. Normally (and at the time of writing), this is run automatically by Github Actions, which generates the relevant log files. However, it is possible to run it manually using the CLI tool
 
+```bash
+sage --python ./lmfdb/tests/generate_snippet_tests.py  -h
 ```
-<table class="ntdata">
-  <thead><tr><td>...</td></tr></thead>
-
-  <tbody>
-   <tr class="odd"> <td>...</td></tr>
-   <tr class="even"><td>...</td></tr>
-   <tr class="odd"> <td>...</td></tr>
-   ...
-  </tbody>
-</table>
+which takes a number of arguments (of which one of the positional arguments `generate` and `test` are required). 
+
+To specify which code snippets should be tested in the Github Action, add a tag of the form 
+
+```yaml
+# e.g. in /lmfdb/number_fields/code.yaml
+...
+snippet_test:                   
+  testQ: 
+    label: 1.1.1.1
+    langs:
+      - sage
+      - magma
+      - oscar
+      - gp
+    url: NumberField/1.1.1.1/download/{lang}
 ```
 
-... we might also switch to CSS3's nth-element selector and forget about this.
+The tag `langs` here is optional, and if omitted, all (available) languages in the `prompt` tag will be used. 
 
+### Snippet test and generate Github Actions
 
-Properties
-----------
+Part of the code snippet testing system is a pair of code actions to generate evaluation log files automatically. The first one runs when a change to a `code*.yaml` file is pulled to the main branch, and if so, regenerates the evaluation files. If the output is different from the previous evaluation files, it will make a pull request, allowing you to manually check that the output is as expected. The logic is in `.github/workflows/snippet_generate.yml`.
 
-The table on the right renders Strings formatted in the following datastructure:
+Additionally, there's a second CI action which runs twice a month and checks that the evaluations are consistent - it can also be run on demand. This ensures that if and when SageMath (or another CAS) deprecates a function, we'll know without having to wait for someone to update the yaml files, rerun the code or submit a bug report. If this action generates a different output than what's stored in the evaluation files, it will error, and upload the diff as an artifact. This is in `.github/workflows/snippet_test.yml`. Furthermore, it will create/update an issue (see https://github.com/LMFDB/lmfdb/issues/6810 for example) which keeps track of all the evaluation errors.
 
-```
-prop = [ ( '<description>', [ '<value 1>', '<value 2>', ...] ), ... ]
-```
 
-or
+Pro Tip: Debugging
+-------------------
 
+Adding
 ```
-prop = [ ( '<description>', '<value>'), ('<description>', '<value>'), ... ]
+  import pdb; pdb.set_trace()
 ```
+somewhere in a python file (e.g. protected inside a sensible if) will pause the execution of the program; 
+this puts you inside an interactive python debugger, where you can:
 
-you can mix list or non-list.
+- check for the local variables with `dir()`, 
+- you can execute arbitrary python code (e.g. to inspect objects),
+- use `pp <var name>` to pretty print variables,
+- continue executing code using the `n` command.
 
+If you get lost, the command `bt` shows you exactly where you
+are, and `up` helps you to get on step up on the stack.
+Of course, `help <command>` will tell you more.
 
-LaTeX Macros
-------------
-
-Latex macros are documented in a knowl that will appear when you start editing one.
+Git Tips
+=========
 
+Contributing to the LMFDB requires some familiarity with git, a popular version control system which LMFDB uses. The LMFDB code base is hosted on Github, which is a common hosting service owned by Microsoft.
+- [Official git tutorial](https://git-scm.com/docs/gittutorial) and [Github's git guide](https://github.com/git-guides)
+- Use a global .gitignore file to exclude temporary files like `.DS_store` or `main.py~`; [copy-paste ready instructions on github](https://help.github.com/articles/ignoring-files#global-gitignore).
+- Configure your global .gitconfig for shortcuts or global configuration options; for example:
+  ```
+[alias]
+		st=status
+		aliases=!git config --get-regexp 'alias.*' | colrm 1 6 | sed 's/[ ]/ = /'
+		ci=commit
+		br=branch
+		co=checkout
+		df=diff
+		who=shortlog -s --
+		ll = log --oneline --graph --decorate -25
+		lla = log --oneline --graph --decorate --all -25
+		wdiff=diff --word-diff=color
+[color]
+	ui = auto
+	branch = auto
+	diff = auto
+	interactive = auto
+	status = auto
+	```
 
 Server Hook
 -----------

lmfdb/abvar/__init__.py

@@ -1,11 +1,9 @@
 from lmfdb.app import app
-from lmfdb.logger import make_logger
 from flask import Blueprint
 
 abvar_page = Blueprint(
     "abvar", __name__, template_folder="templates", static_folder="static"
 )
-abvar_logger = make_logger(abvar_page)
 
 
 @abvar_page.context_processor

lmfdb/abvar/fq/__init__.py

@@ -1,10 +1,8 @@
 
 from lmfdb.app import app
-from lmfdb.logger import make_logger
 from flask import Blueprint
 
 abvarfq_page = Blueprint("abvarfq", __name__, template_folder="templates", static_folder="static")
-abvarfq_logger = make_logger(abvarfq_page)
 
 
 @abvarfq_page.context_processor

lmfdb/abvar/fq/isog_class.py

@@ -12,7 +12,6 @@
 from collections import Counter
 
 from lmfdb.utils import encode_plot, display_float
-from lmfdb.logger import make_logger
 
 from lmfdb import db
 from lmfdb.app import app
@@ -44,9 +43,6 @@ def maxq(g, p):
         return maxgen[g]
 
 
-logger = make_logger("abvarfq")
-
-
 #########################
 #   Label manipulation
 #########################

lmfdb/abvar/fq/main.py

@@ -6,7 +6,6 @@
 from sage.databases.cremona import cremona_letter_code
 
 from lmfdb import db
-from lmfdb.logger import make_logger
 from lmfdb.utils import (
     to_dict, flash_error, integer_options, display_knowl, coeff_to_poly,
     SearchArray, TextBox, TextBoxWithSelect, SkipBox, CheckBox, CheckboxSpacer, YesNoBox,
@@ -25,8 +24,6 @@
 from lmfdb.abvar.fq.download import AbvarFq_download
 from lmfdb.utils.search_parsing import parse_primes
 
-logger = make_logger("abvarfq")
-
 #########################
 #    Top level
 #########################

lmfdb/abvar/fq/stats.py

@@ -4,11 +4,9 @@
 from lmfdb import db
 from lmfdb.utils import comma, display_knowl, StatsDisplay, proportioners, totaler
 from psycodict.utils import range_formatter
-from lmfdb.logger import make_logger
 from sage.misc.lazy_attribute import lazy_attribute
 from sage.misc.cachefunc import cached_method
 
-logger = make_logger("abvarfq")
 def yn(t):
     return "yes" if (t and t != "no") else "no"
 def ynu(t):

lmfdb/api/__init__.py

@@ -1,10 +1,8 @@
 
 from lmfdb.app import app
-from lmfdb.logger import make_logger
 from flask import Blueprint
 
 api_page = Blueprint("API", __name__, template_folder='templates', static_folder="static")
-api_logger = make_logger(api_page)
 
 @api_page.context_processor
 def body_class():