FreeCompetitors/modules/missing.py

# THIS SOFTWARE IS A PART OF FREE COMPETITOR PROJECT
# THE FOLLOWING SOURCE CODE I UNDER THE GNU
# AGPL LICENSE V3 OR ANY LATER VERSION.

# This project is not for simple users, but for
# web-masters and a like, so we are counting on
# your ability to set it up and running.

##################################################

# This file is importart, since it's important to
# keep track of software that the users are
# searching, but not getting any answers to.
# The idea is very simple. If the score of any
# search is below 60%, we add the search term into
# a list. But instead of it being a dumb list, it's
# a smart list.

# Using the similar() function we can group very
# similar terms together. So the users could misspel
# certain names. For example we might group '3D Max'
# '3DSMax' into the same data-structure. Using the
# simple count of how much users spell this or that
# name more often, we will make a suggestion for the
# maintainer. So the maintainer could add the missing
# names into the data. Or at least report them to us.

##################################################

import os
import json

from difflib import SequenceMatcher # checks how similar are two strings

def similar(a, b):
    # I guess it simpifies the syntax for SequenceMatcher
    # In the previous version we use Lavenshtain but it made
    # it an issue for some people to install.
    return SequenceMatcher(None, a, b).ratio()

def add(name):

    # This function will add a datapoint into the missing

    # This shows up way too often. And will show up. Untill we
    # will desing a logo.
    if name == "favicon.ico":
        return
    
    # first we need to make sure that the file exists
    try:
        with open("data/missing.json") as json_file:
            missing = json.load(json_file)

            # Reverse the old file
            if type(missing) == dict:
                missing = []
            
    except:
        missing = []

    # There could be a problem with writing so we look into it
    # for a very close match. Up to about 60%. No more.

    match_missing = 0
    closest_missing = 0
    
    found = False
    for ind, n in enumerate(missing):
        for i in n:
            sim = similar(name, i)
            if sim > 0.6: # At least 60% match
                found = True
                if match_missing < sim:
                    match_missing = sim
                    closest_missing = ind
    if not found:
        missing.append({name:1})
    else:

        if not name in missing[closest_missing]:
            missing[closest_missing][name] = 1
        else:
            missing[closest_missing][name] += 1
        
    # Now we save the file
    with open("data/missing.json", 'w') as f:
        json.dump(missing, f, indent=4, sort_keys=True)

def remove(name):

    # This function will remove a datapoint from the missing

    try:
        with open("data/missing.json") as json_file:
            missing = json.load(json_file)

            # Reverse the old file
            if type(missing) == dict:
                missing = []
            
    except:
        missing = []

    # There could be a problem with writing so we look into it
    # for a very close match. Up to about 60%. No more.

    match_missing = 0
    closest_missing = 0
    
    found = False
    for ind, n in enumerate(missing):
        for i in n:
            sim = similar(name, i)
            if sim > 0.6: # At least 60% match
                found = True
                if match_missing < sim:
                    match_missing = sim
                    closest_missing = ind

    if found:
        del missing[closest_missing]

    # Now we save the file
    with open("data/missing.json", 'w') as f:
        json.dump(missing, f, indent=4, sort_keys=True)


def List():

    # This function will list missing in markdown format

    try:
        with open("data/missing.json") as json_file:
            missing = json.load(json_file)

            # Reverse the old file
            if type(missing) == dict:
                missing = []
            
    except:
        missing = []

    print("| Done | Best Name | Other Names |")
    print("| --- | --- | --- |")
    for i in missing:
        i = sorted(i.items(), key=lambda x:x[1])
        i = dict(i)

        s = "| | **"+list(i.keys())[0]+"**"
        if len(i) > 1:
            s = s + " | "
            for b in i:
                if b == list(i.keys())[0]:
                    continue
                comma = ", "
                if b == list(i.keys())[-1]:
                    comma = ""
                s = s + b + comma
            s = s + " |"
        else:
            s = s + " | |"
        print(s)


def List_html():

    # This function will list missing in html format

    try:
        with open("data/missing.json") as json_file:
            missing = json.load(json_file)

            # Reverse the old file
            if type(missing) == dict:
                missing = []
            
    except:
        missing = []

    page = """

    <h1>Known Missing Software</h1>
    
    <p>This list is auto-generated by the server software when the software
    cannot find a '.json' for for the searched query. It may or may not
    be know to the developers, or even to the server operators. So please report
    the software into <a href="https://notabug.org/jyamihud/FreeCompetitors/issues/25">
    the Missing Software Page</a> in our source code repository.</p>

    """
    
    
    for i in missing:
        i = sorted(i.items(), key=lambda x:x[1])
        i = dict(i)

        if len(i) > 1:
            page = page + "<details><summary>"+list(i.keys())[0]+"</summary>"
            for b in i:
                if b == list(i.keys())[0]:
                    continue
                if b == list(i.keys())[-1]:
                    comma = ""
                page = page + "<span>&nbsp;&nbsp;&nbsp;&nbsp;" + b + "</span><br>"
            page = page + "</details>"
        else:
            page = page + list(i.keys())[0]+"<br>"
        page = page + "\n\n"
        
    return page