[WIP] Add a -H/--hash-function option.

khmer/__init__.py

@@ -88,6 +88,7 @@
 
 _buckets_per_byte = {
     'countgraph': 1,
+    'counttable': 1,
     'smallcountgraph': 2,
     'nodegraph': 8,
 }

khmer/khmer_args.py

@@ -54,7 +54,7 @@
 from khmer import extract_countgraph_info
 from khmer import __version__
 from .utils import print_error
-from .khmer_logger import log_info, log_warn, configure_logging
+from .khmer_logger import log_info, log_warn, configure_logging, log_error
 
 
 DEFAULT_K = 32
@@ -237,6 +237,9 @@ def check_conflicting_args(args, hashtype):
     loadgraph_autoarg_conflicts = ("unique_kmers", "max_memory_usage")
 
     if getattr(args, "loadgraph", None):
+        if getattr(args, "hash_function", 'twobit-exact') != 'twobit-exact':
+            log_error('ERROR: cannot load different hash functions yet.')
+            sys.exit(1)
 
         # check for table config args
         for key, value in loadgraph_table_conflicts.items():
@@ -467,6 +470,10 @@ def build_counting_args(descr=None, epilog=None, citations=None):
     parser.add_argument('--small-count', default=False, action='store_true',
                         help='Reduce memory usage by using a smaller counter'
                         ' for individual kmers.')
+    parser.add_argument('-H', '--hash-function', type=str,
+                       default='twobit-exact',
+                       help='choose hash function to use: twobit-exact, ' +
+                       'murmur')
 
     return parser
 
@@ -547,18 +554,80 @@ def create_countgraph(args, ksize=None, multiplier=1.0, fp_rate=0.1):
 
     if ksize is None:
         ksize = args.ksize
+
     if ksize > 32:
-        print_error("\n** ERROR: khmer only supports k-mer sizes <= 32.\n")
+        print_error("\n** ERROR: khmer only supports k-mer sizes <= 32 for "
+                    "graphs.\n")
+        sys.exit(1)
+
+    if args.hash_function != 'twobit-exact':
+        print_error("\n** ERROR: graphs only support hash function (-H) "
+                    "'twobit-exact'.")
+        sys.exit(1)
+
+    if args.small_count:
+        tabletype = 'smallcountgraph'
+        tableclass = khmer.SmallCountgraph
+    else:
+        tabletype = 'countgraph'
+        tableclass = khmer.Countgraph
+
+    tablesize = calculate_graphsize(args, tabletype, multiplier=multiplier)
+    return khmer.Countgraph(ksize, tablesize, args.n_tables)
+
+
+def create_counttable(args, ksize=None, multiplier=1.0, fp_rate=0.1):
+    """Create and return a counttable."""
+    args = _check_fp_rate(args, fp_rate)
+
+    if hasattr(args, 'force'):
+        if args.n_tables > 20:
+            if not args.force:
+                print_error(
+                    "\n** ERROR: khmer only supports number "
+                    "of tables <= 20.\n")
+                sys.exit(1)
+            else:
+                if args.n_tables > 20:
+                    log_warn("\n*** Warning: Maximum recommended number of "
+                             "tables is 20, discarded by force nonetheless!\n")
+
+    if ksize is None:
+        ksize = args.ksize
+
+    if args.hash_function == 'twobit-exact':
+        if ksize > 32:
+            print_error("\n** ERROR: hash function 'twobit-exact' only " +
+                        "supports k-mer sizes <= 32.")
+            print_error("** See -H/--hash-function for alternatives.")
+            sys.exit(1)
+    elif args.hash_function == 'murmur':
+        pass
+    else:
+        print_error("\n** Error, unknown hash function")
         sys.exit(1)
 
     if args.small_count:
+        if args.hash_function == 'murmur':
+            print_error("\n** ERROR: hash function 'murmur' does not " +
+                        "support small counts yet.")
+            sys.exit(1)
+
         tablesize = calculate_graphsize(args, 'smallcountgraph',
                                         multiplier=multiplier)
         return khmer.SmallCountgraph(ksize, tablesize, args.n_tables)
     else:
-        tablesize = calculate_graphsize(args, 'countgraph',
-                                        multiplier=multiplier)
-        return khmer.Countgraph(ksize, tablesize, args.n_tables)
+        if args.hash_function == 'murmur':
+            tabletype = 'counttable'
+            tableclass = khmer.Counttable
+        elif args.hash_function == 'twobit-exact':
+            tabletype = 'countgraph'
+            tableclass = khmer.Countgraph
+        else:
+            assert 0
+
+        tablesize = calculate_graphsize(args, tabletype, multiplier=multiplier)
+        return tableclass(ksize, tablesize, args.n_tables)
 
 
 def report_on_config(args, graphtype='countgraph'):

scripts/filter-abund-single.py

@@ -126,13 +126,17 @@ def main():
     check_space([args.datafile], args.force)
 
     if args.savegraph:
+        if args.hash_function != 'twobit-exact':
+            log_error('ERROR: cannot save different hash functions yet.')
+            sys.exit(1)
+
         tablesize = calculate_graphsize(args, 'countgraph')
         check_space_for_graph(args.savegraph, tablesize, args.force)
 
     report_on_config(args)
 
     log_info('making countgraph')
-    graph = khmer_args.create_countgraph(args)
+    graph = khmer_args.create_counttable(args)
 
     # first, load reads into graph
     rparser = khmer.ReadParser(args.datafile)

scripts/normalize-by-median.py

@@ -330,6 +330,10 @@ def main():  # pylint: disable=too-many-branches,too-many-statements
     check_valid_file_exists(args.input_filenames)
     check_space(args.input_filenames, args.force)
     if args.savegraph:
+        if args.hash_function != 'twobit-exact':
+            log_error('ERROR: cannot save different hash functions yet.')
+            sys.exit(1)
+
         graphsize = calculate_graphsize(args, 'countgraph')
         check_space_for_graph(args.savegraph, graphsize, args.force)
 
@@ -340,7 +344,7 @@ def main():  # pylint: disable=too-many-branches,too-many-statements
         countgraph = khmer.load_countgraph(args.loadgraph)
     else:
         log_info('making countgraph')
-        countgraph = khmer_args.create_countgraph(args)
+        countgraph = khmer_args.create_counttable(args)
 
     # create an object to handle diginorm of all files
     norm = Normalizer(args.cutoff, countgraph)

scripts/trim-low-abund.py

@@ -303,6 +303,10 @@ def main():
     check_valid_file_exists(args.input_filenames)
     check_space(args.input_filenames, args.force)
     if args.savegraph:
+        if args.hash_function != 'twobit-exact':
+            log_error('ERROR: cannot save different hash functions yet.')
+            sys.exit(1)
+
         graphsize = calculate_graphsize(args, graphtype)
         check_space_for_graph(args.savegraph, graphsize, args.force)
 
@@ -317,7 +321,7 @@ def main():
         ct = khmer.load_countgraph(args.loadgraph, args.small_count)
     else:
         log_info('making countgraph')
-        ct = khmer_args.create_countgraph(args)
+        ct = khmer_args.create_counttable(args)
 
     K = ct.ksize()
     tempdir = tempfile.mkdtemp('khmer', 'tmp', args.tempdir)

setup.py

@@ -227,7 +227,7 @@ def check_for_openmp():
         "packages": ['khmer', 'khmer.tests', 'oxli'],
         "package_dir": {'khmer.tests': 'tests'},
         "install_requires": ['screed >= 0.9', 'bz2file'],
-        "setup_requires": ["pytest-runner>=2.0,<3dev"],
+#        "setup_requires": ["pytest-runner>=2.0,<3dev"],
         "extras_require": {':python_version=="2.6"': ['argparse>=1.2.1'],
                            'docs': ['sphinx', 'sphinxcontrib-autoprogram'],
                            'tests': ['pytest>=2.9'],

tests/test_filter_abund.py

@@ -186,6 +186,39 @@ def test_filter_abund_1_singlefile():
     assert 'GGTTGACGGGGCTCAGGG' in seqs
 
 
+def test_filter_abund_1_singlefile_long_k():
+    infile = utils.copy_test_data('test-abund-read-2.fa')
+    in_dir = os.path.dirname(infile)
+
+    script = 'filter-abund-single.py'
+    args = ['-x', '1e7', '-N', '2', '-k', '35', '-H', 'murmur', infile]
+    (status, out, err) = utils.runscript(script, args, in_dir)
+
+    assert 'Total number of unique k-mers: 80' in err, err
+
+    outfile = infile + '.abundfilt'
+    assert os.path.exists(outfile), outfile
+
+    seqs = set([r.sequence for r in screed.open(outfile)])
+    assert len(seqs) == 0
+
+
+def test_filter_abund_1_singlefile_long_k_nosave():
+    infile = utils.copy_test_data('test-abund-read-2.fa')
+    in_dir = os.path.dirname(infile)
+
+    script = 'filter-abund-single.py'
+    args = ['-x', '1e7', '-N', '2', '-k', '35', '-H', 'murmur', infile,
+            '--savegraph', 'foo']
+    (status, out, err) = utils.runscript(script, args, in_dir, fail_ok=True)
+
+    print(out)
+    print(err)
+
+    assert status == 1
+    assert 'ERROR: cannot save different hash functions yet.' in err
+
+
 def test_filter_abund_2_singlefile():
     infile = utils.copy_test_data('test-abund-read-2.fa')
     in_dir = os.path.dirname(infile)

tests/test_normalize_by_median.py

@@ -779,3 +779,67 @@ def test_normalize_by_median_outfile_closed_err():
     (status, out, err) = utils.runscript(script, args)
     assert status == 0, (out, err)
     assert os.path.exists(outfile)
+
+
+def test_normalize_by_median_long_k():
+    CUTOFF = '2'
+
+    infile = utils.copy_test_data('test-abund-read-2.fa')
+    in_dir = os.path.dirname(infile)
+
+    script = 'normalize-by-median.py'
+    args = ['-C', CUTOFF, '-k', '33', '-H', 'murmur', infile]
+    utils.runscript(script, args, in_dir)
+
+    outfile = infile + '.keep'
+    assert os.path.exists(outfile), outfile
+
+    seqs = [r.sequence for r in screed.open(outfile)]
+    assert len(seqs) == 1, seqs
+    assert seqs[0].startswith('GGTTGACGGGGCTCAGGGGG'), seqs
+
+
+def test_normalize_by_median_long_k_twobit_fails():
+    CUTOFF = '2'
+
+    infile = utils.copy_test_data('test-abund-read-2.fa')
+    in_dir = os.path.dirname(infile)
+
+    script = 'normalize-by-median.py'
+    args = ['-C', CUTOFF, '-k', '33', '-H', 'murmur', infile,
+            '-H', 'twobit-exact']
+    (status, out, err) = utils.runscript(script, args, in_dir, fail_ok=True)
+
+    assert status == 1
+    assert "'twobit-exact' only supports k-mer sizes <= 32" in err
+
+
+def test_normalize_by_median_long_k_save_fails():
+    CUTOFF = '2'
+
+    infile = utils.copy_test_data('test-abund-read-2.fa')
+    in_dir = os.path.dirname(infile)
+
+    script = 'normalize-by-median.py'
+    args = ['-C', CUTOFF, '-k', '33', '-H', 'murmur', infile, '-s', 'foo']
+    (status, out, err) = utils.runscript(script, args, in_dir, fail_ok=True)
+
+    assert status == 1
+    assert 'ERROR: cannot save different hash functions yet.' in err
+
+
+def test_normalize_by_median_long_k_load_fails():
+    CUTOFF = '2'
+
+    infile = utils.copy_test_data('test-abund-read-2.fa')
+    in_dir = os.path.dirname(infile)
+
+    script = 'normalize-by-median.py'
+    args = ['-C', CUTOFF, '-k', '33', '-H', 'murmur', infile, '-l', 'foo']
+    (status, out, err) = utils.runscript(script, args, in_dir, fail_ok=True)
+
+    print(out)
+    print(err)
+
+    assert status == 1
+    assert 'ERROR: cannot load different hash functions yet.' in err