searx_ynh/sources/searx/engines/duckduckgo_definitions.py

import json
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text

url = 'https://api.duckduckgo.com/?{query}&format=json&pretty=0&no_redirect=1&d=1'

def result_to_text(url, text, htmlResult):
    # TODO : remove result ending with "Meaning" or "Category"
    dom = html.fromstring(htmlResult)
    a = dom.xpath('//a')
    if len(a)>=1:
        return extract_text(a[0])
    else:
        return text

def html_to_text(htmlFragment):
    dom = html.fromstring(htmlFragment)
    return extract_text(dom)

def request(query, params):
    # TODO add kl={locale}
    params['url'] = url.format(query=urlencode({'q': query}))
    return params


def response(resp):
    search_res = json.loads(resp.text)
    results = []

    content = ''
    heading = search_res.get('Heading', '')
    attributes = []
    urls = []
    infobox_id = None
    relatedTopics = []

    # add answer if there is one
    answer = search_res.get('Answer', '')
    if answer != '':
        results.append({ 'answer' : html_to_text(answer) })

    # add infobox
    if 'Definition' in search_res:
        content = content + search_res.get('Definition', '') 

    if 'Abstract' in search_res:
        content = content + search_res.get('Abstract', '')


    # image
    image = search_res.get('Image', '')
    image = None if image == '' else image

    # attributes
    if 'Infobox' in search_res:
        infobox = search_res.get('Infobox', None)
        if  'content' in infobox:
            for info in infobox.get('content'):
                attributes.append({'label': info.get('label'), 'value': info.get('value')})

    # urls
    for ddg_result in search_res.get('Results', []):
        if 'FirstURL' in ddg_result:
            firstURL = ddg_result.get('FirstURL', '')
            text = ddg_result.get('Text', '')
            urls.append({'title':text, 'url':firstURL})
            results.append({'title':heading, 'url': firstURL})

    # related topics
    for ddg_result in search_res.get('RelatedTopics', None):
        if 'FirstURL' in ddg_result:
            suggestion = result_to_text(ddg_result.get('FirstURL', None), ddg_result.get('Text', None), ddg_result.get('Result', None))
            if suggestion != heading:
                results.append({'suggestion': suggestion})
        elif 'Topics' in ddg_result:
            suggestions = []
            relatedTopics.append({ 'name' : ddg_result.get('Name', ''), 'suggestions': suggestions })
            for topic_result in ddg_result.get('Topics', []):
                suggestion = result_to_text(topic_result.get('FirstURL', None), topic_result.get('Text', None), topic_result.get('Result', None))
                if suggestion != heading:
                    suggestions.append(suggestion)

    # abstract
    abstractURL = search_res.get('AbstractURL', '')
    if abstractURL != '':
        # add as result ? problem always in english
        infobox_id = abstractURL
        urls.append({'title': search_res.get('AbstractSource'), 'url': abstractURL})

    # definition
    definitionURL = search_res.get('DefinitionURL', '')
    if definitionURL != '':
        # add as result ? as answer ? problem always in english
        infobox_id = definitionURL
        urls.append({'title': search_res.get('DefinitionSource'), 'url': definitionURL})

    # entity
    entity = search_res.get('Entity', None)
    # TODO continent / country / department / location / waterfall / mountain range : link to map search, get weather, near by locations
    # TODO musician : link to music search
    # TODO concert tour : ??
    # TODO film / actor / television  / media franchise : links to IMDB / rottentomatoes (or scrap result)
    # TODO music : link tu musicbrainz / last.fm
    # TODO book : ??
    # TODO artist / playwright : ??
    # TODO compagny : ??
    # TODO software / os : ??
    # TODO software engineer : ??
    # TODO prepared food : ??
    # TODO website : ??
    # TODO performing art : ??
    # TODO prepared food : ??
    # TODO programming language : ??
    # TODO file format : ??

    if len(heading)>0:
        # TODO get infobox.meta.value where .label='article_title'
        if image==None and len(attributes)==0 and len(urls)==1 and len(relatedTopics)==0 and len(content)==0:
            results.append({
                    'url': urls[0]['url'],
                    'title': heading,
                    'content': content
                    })
        else:
            results.append({
                    'infobox': heading,
                    'id': infobox_id,
                    'entity': entity,
                    'content': content,
                    'img_src' : image,
                    'attributes': attributes,
                    'urls': urls,
                    'relatedTopics': relatedTopics
                    })

    return results