ZacharyZhang-NY
77219c930b
Ground truth comes only from third-party annotations (MMLongBench-Doc
evidence_page_ids, VisRAG qrels, ScreenSpot-v2 bboxes, Video-MME QA,
LongMemEval answer_session_ids, CodeSearchNet func_path+func_name) and
from document structure; no model judgement is used anywhere.
- bench/corpus/manifest.jsonl: 28 sources with url, retrieved_at, sha256,
annotation_source, license, files[]; verified by bench.manifest_check
- bench/sets/{pdf,image,ui_screenshot,video,code,conversation}.{jsonl} plus
corpus indexes and a manifest that freezes every file's sha256
- 2,928 answerable + 2,380 unanswerable queries over 854 holders; every
query carries scope {holder, scenario, as_of} for transactive routing
- bench/validate_sets.py fails on missing gold, holder mismatch between
gold and query, gold observed after as_of, and manifest digest drift
960 lines
25 KiB
JSON
960 lines
25 KiB
JSON
{
|
|
"corpus_manifest": "corpus/manifest.jsonl",
|
|
"corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347",
|
|
"corpus_records": 50789,
|
|
"holders": 854,
|
|
"modalities": {
|
|
"code": {
|
|
"corpus_file": "corpus/code.jsonl",
|
|
"corpus_records": 21976,
|
|
"corpus_sha256": "0c7a0d220e1f87cd497acf376c0371790abbfdbdaf727916e13e766c934df5cc",
|
|
"holders": [
|
|
"20tab-twentytab-treeeditor",
|
|
"3DLIRIOUS-MeshLabXML",
|
|
"4degrees-riffle",
|
|
"5j9-wikitextparser",
|
|
"5monkeys-django-bananas",
|
|
"AartGoossens-goldencheetahlib",
|
|
"AkihikoITOH-capybara",
|
|
"Alignak-monitoring-contrib-alignak-backend-client",
|
|
"AnthonyMRios-pymetamap",
|
|
"Azure-Azure-MachineLearning-ClientLibrary-Python",
|
|
"Azure-azure-sdk-for-python",
|
|
"BD2KGenomics-toil-lib",
|
|
"BHSPitMonkey-vmflib",
|
|
"BlueDragonX-detach",
|
|
"CITGuru-PyInquirer",
|
|
"CLARIAH-iribaker",
|
|
"Chilipp-sphinx-nbexamples",
|
|
"ChrisBeaumont-soupy",
|
|
"CiscoDevNet-webexteamssdk",
|
|
"Clinical-Genomics-scout",
|
|
"ContinuumIO-pycosat",
|
|
"Crypto-toolbox-btfxwss",
|
|
"Danielhiversen-PyXiaomiGateway",
|
|
"Dirrot-python-cryptocoincharts-api",
|
|
"DiscordBotList-DBL-Python-Library",
|
|
"EpistasisLab-scikit-mdr",
|
|
"Erotemic-progiter",
|
|
"Erotemic-xdoctest",
|
|
"GGiecold-Density_Sampling",
|
|
"Gandi-pyramid_kvs",
|
|
"GeoffAtHome-lightwave",
|
|
"GreatFruitOmsk-tailhead",
|
|
"HaaLeo-pylint-file-header",
|
|
"HumanBrainProject-hbp-service-client",
|
|
"HydraChain-hydrachain",
|
|
"IEMLdev-ieml",
|
|
"IdentityPython-fedoidcmsg",
|
|
"JIC-CSB-jicbioimage.illustrate",
|
|
"Jasily-jasily-python",
|
|
"Jaymon-pyt",
|
|
"Jaza-s3-saver",
|
|
"JonathanRaiman-ciseau",
|
|
"JukeboxPipeline-jukedj",
|
|
"Julian-Ivoire",
|
|
"Julian-Virtue",
|
|
"Kroisse-FormEncode-Jinja2",
|
|
"L3viathan-sql-mojo-parser",
|
|
"LLNL-scraper",
|
|
"LionelAuroux-pyrser",
|
|
"LordSputnik-mutagen",
|
|
"Loudr-asana-hub",
|
|
"LudovicRousseau-PyKCS11",
|
|
"LuminosoInsight-luminoso-api-client-python",
|
|
"ManageIQ-manageiq-api-client-python",
|
|
"MartinHjelmare-leicacam",
|
|
"MaxStrange-AudioSegment",
|
|
"MeirKriheli-django-bidi-utils",
|
|
"MeirKriheli-python-bidi",
|
|
"Microsoft-botbuilder-python",
|
|
"MinchinWeb-minchin.text",
|
|
"MisterY-price-database",
|
|
"MitchellChu-torndsession",
|
|
"MomsFriendlyRobotCompany-fake_rpi",
|
|
"MuhammedHasan-sklearn_utils",
|
|
"Music-Moo-music2storage",
|
|
"MycroftAI-padaos",
|
|
"NLeSC-pycoeman",
|
|
"NarrativeScience-lsi",
|
|
"NateFerrero-oauth2lib",
|
|
"Nekmo-amazon-dash",
|
|
"Nekroze-greencard",
|
|
"Nic30-hwt",
|
|
"Numergy-yoda",
|
|
"ONSdigital-sdc-cryptography",
|
|
"OldhamMade-exemelopy",
|
|
"OpenKMIP-PyKMIP",
|
|
"OpenMath-py-openmath",
|
|
"Overdrivr-pytelemetry",
|
|
"Patreon-multiget-cache-py",
|
|
"PierreRaybaut-PyQtdoc",
|
|
"Pirionfr-pyLinky",
|
|
"PolicyStat-docx2html",
|
|
"Polyconseil-devpi-metawheel",
|
|
"PyCQA-pylint",
|
|
"PyFilesystem-s3fs",
|
|
"PyProphet-pyprophet",
|
|
"PythonSanSebastian-docstamp",
|
|
"Qiskit-qiskit-terra",
|
|
"RRZE-HPC-kerncraft",
|
|
"RazerM-yourls-python",
|
|
"SeabornGames-Logger",
|
|
"SectorLabs-django-postgres-extra",
|
|
"SiLab-Bonn-pixel_clusterizer",
|
|
"SketchingDev-Doodle-Dashboard",
|
|
"Skyscanner-pages",
|
|
"SmBe19-praw-OAuth2Util",
|
|
"SmartDeveloperHub-sdh-curator-py",
|
|
"SmokinCaterpillar-pypet",
|
|
"StackSentinel-stacksentinel-python",
|
|
"StaticCube-python-synology",
|
|
"TDG-Platform-cloud-harness",
|
|
"Terrance-SkPy",
|
|
"Tivix-django-rest-auth",
|
|
"Toblerity-rtree",
|
|
"ToucanToco-toucan-data-sdk",
|
|
"Trim21-sdu_bkjws",
|
|
"Tsjerk-Insane",
|
|
"UCBerkeleySETI-blimpy",
|
|
"UDST-pandana",
|
|
"URXtech-cmph-cffi",
|
|
"Valuehorizon-valuehorizon-forex",
|
|
"VingtCinq-python-resize-image",
|
|
"VorskiImagineering-C3PO",
|
|
"WGBH-wagtail-streamfieldtools",
|
|
"WhoopInc-pytsdb",
|
|
"WoLpH-mt940",
|
|
"Yaco-Sistemas-django-transmeta",
|
|
"Yelp-py_zipkin",
|
|
"Yubico-python-u2flib-host",
|
|
"ZELLMECHANIK-DRESDEN-fcswrite",
|
|
"Zsailer-pyasr",
|
|
"a-tal-nagaram",
|
|
"abau171-highfive",
|
|
"accraze-pymtranslate",
|
|
"adafruit-Adafruit_Python_PureIO",
|
|
"addok-addok",
|
|
"admiralobvious-flask-mysqldb",
|
|
"agabrown-PyGaia",
|
|
"agile-geoscience-striplog",
|
|
"aholkner-bacon",
|
|
"albertodonato-prometheus-aioexporter",
|
|
"alexprengere-currencyconverter",
|
|
"alexseitsinger-find_best_string",
|
|
"alghafli-fileslice",
|
|
"algofairness-BlackBoxAuditing",
|
|
"alimuldal-PyFNND",
|
|
"allanlei-django-argparse-command",
|
|
"alpacahq-alpaca-trade-api-python",
|
|
"ambitioninc-python-logentries-api",
|
|
"amcat-progressmonitor",
|
|
"amelchio-eternalegypt",
|
|
"amorison-loam",
|
|
"amyth-django-instapush",
|
|
"analytehealth-django-analytics",
|
|
"andrefsp-pyflot",
|
|
"andreikop-python-ws-discovery",
|
|
"anentropic-dirtyebay",
|
|
"ankitmathur3193-song-cli",
|
|
"ansible-ansible-lint",
|
|
"antevens-listen",
|
|
"apache-airflow",
|
|
"appier-h5sparse",
|
|
"appknox-google-chartwrapper",
|
|
"arlyon-hyperion",
|
|
"arve0-microscopestitching",
|
|
"asciimoo-searx",
|
|
"ashleysommer-sanic-cors",
|
|
"assemblerflow-flowcraft",
|
|
"astrofrog-numtraits",
|
|
"astrorafael-twisted-mqtt",
|
|
"athre0z-wasm",
|
|
"atlassistant-pychatl",
|
|
"attm2x-m2x-python",
|
|
"audiencepi-SimilarWeb-Python",
|
|
"ausaki-python-validator",
|
|
"authomatic-authomatic",
|
|
"avelino-bottle-auth",
|
|
"awentzonline-keras-vgg-buddy",
|
|
"bakwc-PySyncObj",
|
|
"benhoff-microphone",
|
|
"bennylope-django-taggit-labels",
|
|
"bheinzerling-descriptors",
|
|
"bicv-LogGabor",
|
|
"bigdatacesga-kvstore",
|
|
"bindlock-vklancer",
|
|
"bipsandbytes-django-api",
|
|
"bitlabstudio-django-dashboard-app",
|
|
"bitlabstudio-django-hero-slider",
|
|
"bivab-smbus-cffi",
|
|
"bjodah-pycvodes",
|
|
"bjoernricks-python-quilt",
|
|
"bkg-greenwich",
|
|
"bkosciow-python_iot-1",
|
|
"blockstack-packages-keychain-manager-py",
|
|
"blockstack-zone-file-py",
|
|
"bloomreach-s4cmd",
|
|
"bluedisk-hangul-toolkit",
|
|
"bmaeser-pyque",
|
|
"bolt-project-bolt",
|
|
"boto-s3transfer",
|
|
"boundary-pulse-api-cli",
|
|
"brandon-rhodes-python-jplephem",
|
|
"bruth-cypher",
|
|
"bsmithyman-problemo",
|
|
"bucknerns-concordance",
|
|
"bunchesofdonald-photohash",
|
|
"buzzfeed-phonon",
|
|
"c-soft-satel_integra",
|
|
"caktus-rapidsms-twilio",
|
|
"calston-tensor",
|
|
"capitalone-giraffez",
|
|
"cartoonist-pystream-protobuf",
|
|
"cassinyio-SwarmSpawner",
|
|
"ccubed-Shosetsu",
|
|
"celiao-tmdbsimple",
|
|
"cgarciae-dicto",
|
|
"chairco-pyRscript",
|
|
"chaoss-grimoirelab-kingarthur",
|
|
"chaoss-grimoirelab-perceval",
|
|
"chaoss-grimoirelab-perceval-mozilla",
|
|
"chrisjrn-registrasion",
|
|
"chrisnorman7-wxgoodies",
|
|
"chrisrink10-basilisp",
|
|
"chuck1-codemach",
|
|
"cloud9ers-gurumate",
|
|
"clouserw-tower",
|
|
"cmcginty-PyWeather",
|
|
"cnu-scrapy-random-useragent",
|
|
"codeghar-brokerlso",
|
|
"codeinthehole-django-async-messages",
|
|
"codelv-enaml-web",
|
|
"coleifer-irc",
|
|
"collab-project-django-admin-footer",
|
|
"connectordb-connectordb-python",
|
|
"coops-r53",
|
|
"crankycoder-hydra",
|
|
"cthoyt-ols-client",
|
|
"cyberdelia-metrology",
|
|
"d0ugal-home",
|
|
"dagster-io-dagster",
|
|
"datacamp-pythonwhat",
|
|
"dave-shawley-ietfparse",
|
|
"davidmiller-urlhelp",
|
|
"dbcli-cli_helpers",
|
|
"deepmipt-DeepPavlov",
|
|
"dessibelle-sorl-thumbnail-serializer-field",
|
|
"dialoguemd-multi-method",
|
|
"disqus-python-phabricator",
|
|
"django-inplaceedit-django-inplaceedit",
|
|
"djangobot-djangobot",
|
|
"dogoncouch-lightcli",
|
|
"domainaware-parsedmarc",
|
|
"dopefishh-pympi",
|
|
"dossier-dossier.store",
|
|
"draios-python-sdc-client",
|
|
"dreidev-Suggestions",
|
|
"drestebon-papageorge",
|
|
"drewsonne-pyum",
|
|
"drhagen-parsita",
|
|
"dropseed-combine",
|
|
"dry-python-dependencies",
|
|
"dsoprea-pytzPure",
|
|
"dwavesystems-minorminer",
|
|
"dwhswenson-autorelease",
|
|
"eagleflo-adjutant",
|
|
"eclarke-melt",
|
|
"ecometrica-grandfatherson",
|
|
"edavis-django-override-settings",
|
|
"edoburu-django-debugtools",
|
|
"edoburu-django-tag-parser",
|
|
"edx-ecommerce-worker",
|
|
"edx-web-fragments",
|
|
"eecs-autograder-autograder-sandbox",
|
|
"egh-ledger-autosync",
|
|
"eisber-sarplus",
|
|
"ekmmetering-ekmmeters",
|
|
"ekzhu-datasketch",
|
|
"ellisonleao-vl",
|
|
"elliterate-capybara.py",
|
|
"ellmetha-neojsonrpc",
|
|
"elyase-masstable",
|
|
"eng-tools-geofound",
|
|
"erichiggins-gaek",
|
|
"ethereum-py_ecc",
|
|
"etsinko-pymonoprice",
|
|
"eyeseast-python-frontmatter",
|
|
"farzadghanei-statsd-metrics",
|
|
"fastmonkeys-stellar",
|
|
"fatihsucu-pyzomato",
|
|
"feifangit-dj-api-auth",
|
|
"ferrix-dj-mongohq-url",
|
|
"filepreviews-filepreviews-python",
|
|
"flit-elapsedtimer",
|
|
"fm4d-PyMarkovTextGenerator",
|
|
"frascoweb-easywebassets",
|
|
"frispete-keyrings.cryptfile",
|
|
"fukuball-fuku-ml",
|
|
"funilrys-PyFunceble",
|
|
"garyelephant-pygrok",
|
|
"gawel-panoramisk",
|
|
"gears-gears",
|
|
"genialis-genesis-pyapi",
|
|
"geopython-OWSLib",
|
|
"getgauge-gauge-python",
|
|
"getsentry-sentry-hipchat-ac",
|
|
"ghcollin-multitables",
|
|
"gholt-swiftly",
|
|
"giffels-CloudStackAIO",
|
|
"globality-corp-flake8-logging-format",
|
|
"glottobank-python-newick",
|
|
"gmr-tornado-elasticsearch",
|
|
"gnosis-gnosis-py",
|
|
"google-brotli",
|
|
"google-gin-config",
|
|
"google-pybadges",
|
|
"gooofy-py-nltools",
|
|
"grampajoe-happy",
|
|
"graphql-python-graphene-sqlalchemy",
|
|
"greenbender-pynntp",
|
|
"griffithlab-civicpy",
|
|
"gtaylor-python-route53",
|
|
"gtzampanakis-aml",
|
|
"gunyarakun-python-shogi",
|
|
"gvalkov-tornado-http-auth",
|
|
"h2non-pook",
|
|
"h2oai-h2o-3",
|
|
"hamelsmu-ktext",
|
|
"hdima-erlport",
|
|
"hipikat-adjspecies",
|
|
"hiposfer-o2g",
|
|
"hirmeos-entity-fishing-client-python",
|
|
"hivetech-dna",
|
|
"horazont-aiosasl",
|
|
"howie6879-ruia",
|
|
"hustlzp-Flask-Boost",
|
|
"hydrosquall-tiingo-python",
|
|
"ianare-django-memcache-admin",
|
|
"iandees-pyosm",
|
|
"ibm-watson-iot-iot-python",
|
|
"idlesign-django-etc",
|
|
"idlesign-django-sitetree",
|
|
"idlesign-django-yaturbo",
|
|
"impulsare-config",
|
|
"incuna-django-pgcrypto-fields",
|
|
"incuna-django-user-management",
|
|
"incuna-django-wkhtmltopdf",
|
|
"ingolemo-python-lenses",
|
|
"instagrambot-instabot",
|
|
"intel-analytics-BigDL",
|
|
"inveniosoftware-counter-robots",
|
|
"inveniosoftware-invenio-migrator",
|
|
"inveniosoftware-invenio-records-ui",
|
|
"ioam-parambokeh",
|
|
"ionelmc-python-tblib",
|
|
"iotaledger-iota.lib.py",
|
|
"iotile-typedargs",
|
|
"isambard-uob-budeff",
|
|
"its-rigs-Trolly",
|
|
"jab-bidict",
|
|
"jaraco-jaraco.classes",
|
|
"jaraco-jaraco.itertools",
|
|
"jaraco-jaraco.util",
|
|
"jaraco-svg.charts",
|
|
"jaraco-wolframalpha",
|
|
"jazzband-django-ddp",
|
|
"jdrumgoole-pymongo_formatter",
|
|
"jerith-txfake",
|
|
"jfialkoff-pyphaxio",
|
|
"jgorset-django-kronos",
|
|
"jic-dtool-dtool-azure",
|
|
"jic-dtool-dtool-s3",
|
|
"jjangsangy-py-translate",
|
|
"joeblackwaslike-pricing",
|
|
"jonashaag-django-addanother",
|
|
"jonfaustman-django-frontend",
|
|
"joopert-nad_receiver",
|
|
"josiahcarlson-rom",
|
|
"jp74-django-model-publisher",
|
|
"jrfonseca-xdot.py",
|
|
"jrxFive-python-nomad",
|
|
"jshiv-turntable",
|
|
"jtambasco-opticalmaterialspy",
|
|
"jterrace-pyssim",
|
|
"juga0-dhcpcanon",
|
|
"jupyter-widgets-jupyterlab-sidecar",
|
|
"justanr-flask-allows",
|
|
"justbilt-untp",
|
|
"kadrlica-pymodeler",
|
|
"kamalshadi-Localization",
|
|
"karimbahgat-PyCRS",
|
|
"katerina7479-pypdflite",
|
|
"keflavich-plfit",
|
|
"kejbaly2-comdev",
|
|
"kenneth-reitz-dynamo",
|
|
"kennethreitz-flask-common",
|
|
"kennethreitz-twitter-scraper",
|
|
"kgiusti-pyngus",
|
|
"kmmbvnr-django-any",
|
|
"kolypto-py-smsframework",
|
|
"ktdreyer-rhcalendar",
|
|
"kurtbrose-sampro",
|
|
"kxepal-viivakoodi",
|
|
"labstreaminglayer-liblsl-Python",
|
|
"lambdalisue-django-codemirror-widget",
|
|
"laplacesdemon-django-youtube",
|
|
"larsyencken-proj",
|
|
"lcharleux-argiope",
|
|
"ldesousa-hex-utils",
|
|
"leVirve-dcard-spider",
|
|
"lensacom-sparkit-learn",
|
|
"leovp-steamfiles",
|
|
"lepture-flask-oauthlib",
|
|
"librosa-librosa",
|
|
"lincolnloop-goodconf",
|
|
"linjackson78-jstyleson",
|
|
"linuxwhatelse-mapper",
|
|
"litl-leeroy",
|
|
"littlemo-mohand",
|
|
"liubola-sangreal-db",
|
|
"lkreidberg-batman",
|
|
"llimllib-pub",
|
|
"lmjohns3-theanets",
|
|
"loanzen-probe-py",
|
|
"lord63-five.py",
|
|
"lsst-sqre-ltd-conveyor",
|
|
"lucaslamounier-USGSDownload",
|
|
"lvh-txampext",
|
|
"makearl-tornado-profile",
|
|
"makeev-django-boolean-switch",
|
|
"makinacorpus-django-templatetag-handlebars",
|
|
"manrajgrover-py-spinners",
|
|
"maralla-bmd",
|
|
"marcoagner-Flask-QRcode",
|
|
"marians-py-daterangestr",
|
|
"marrabld-planarradpy",
|
|
"marrow-package",
|
|
"marrow-web.db",
|
|
"marshmallow-code-marshmallow-sqlalchemy",
|
|
"matrix-org-python-canonicaljson",
|
|
"matthewdeanmartin-find_known_secrets",
|
|
"maxcountryman-flask-uploads",
|
|
"may-day-wsgicors",
|
|
"mblayman-httpony",
|
|
"mcs07-MolVS",
|
|
"mdgoldberg-sportsref",
|
|
"memsql-memsql-python",
|
|
"mental32-spotify.py",
|
|
"merrychap-shellen",
|
|
"mesos-magellan-pyrallelsa",
|
|
"mezuro-kalibro_client_py",
|
|
"mfcovington-django-project-home-templatetags",
|
|
"mikewaters-command-session",
|
|
"mjtamlyn-django-cbvpatterns",
|
|
"mk-fg-python-onedrive",
|
|
"mohabusama-pyguacamole",
|
|
"mohan3d-PyOpenload",
|
|
"mopidy-mopidy-spotify",
|
|
"mordred-descriptor-mordred",
|
|
"moreati-ppeg",
|
|
"mozilla-iot-webthing-python",
|
|
"mozilla-services-sign-xpi-lib",
|
|
"mpenning-polymer",
|
|
"mrsarm-mongotail",
|
|
"mrtazz-notifo.py",
|
|
"mseclab-PyJFuzz",
|
|
"mshroyer-pointfree",
|
|
"mwarkentin-django-watchman",
|
|
"mwhooker-jsonselect",
|
|
"myint-autoflake",
|
|
"napalm-automation-napalm-yang",
|
|
"nathforge-validatesns",
|
|
"nats-io-asyncio-nats",
|
|
"ncolony-ncolony",
|
|
"ndokter-dsmr_parser",
|
|
"ndrlslz-ternya",
|
|
"neherlab-treetime",
|
|
"neithere-argh",
|
|
"nens-qgispluginreleaser",
|
|
"neogenix-jsonklog",
|
|
"nesdis-djongo",
|
|
"netom-satispy",
|
|
"neurodata-ndio",
|
|
"neurosynth-neurosynth",
|
|
"nicferrier-pyproxyfs",
|
|
"nicfit-nicfit.py",
|
|
"nickw444-flask-ldap3-login",
|
|
"nicolas-van-mailflash",
|
|
"nigma-django-twilio-sms",
|
|
"ninjaaron-libaaron",
|
|
"nixmc-chatter",
|
|
"nkgilley-python-ecobee-api",
|
|
"not-na-peng3d",
|
|
"nprapps-copydoc",
|
|
"nsqio-pynsq",
|
|
"nvdv-vprof",
|
|
"oceanprotocol-oceandb-elasticsearch-driver",
|
|
"ojarva-python-sshpubkeys",
|
|
"ojengwa-accounting",
|
|
"omaciel-pytest-fauxfactory",
|
|
"open-mmlab-mmcv",
|
|
"opencast-pyCA",
|
|
"openergy-oplus",
|
|
"openmicroscopy-yaclifw",
|
|
"openvenues-pypostal",
|
|
"orangain-scrapy-s3pipeline",
|
|
"oriontvv-pyaspeller",
|
|
"oscarbranson-latools",
|
|
"ossobv-exactonline",
|
|
"pasztorpisti-kwonly-args",
|
|
"pelotoncycle-cycle_detector",
|
|
"penguinmenac3-opendatalake",
|
|
"pgjones-hypercorn",
|
|
"phfaist-pylatexenc",
|
|
"philipsoutham-py-mysql2pgsql",
|
|
"phoikoi-sisy",
|
|
"piface-pifacecommon",
|
|
"pltrdy-rouge",
|
|
"pmacosta-peng",
|
|
"pmbarrett314-curses-menu",
|
|
"power-system-simulation-toolbox-psst",
|
|
"praekelt-django-analytics",
|
|
"praekeltfoundation-seed-control-interface",
|
|
"precond-django-customary",
|
|
"proofit404-service-factory",
|
|
"psd-tools-packbits",
|
|
"pyblish-pyblish-maya",
|
|
"pyca-pyopenssl",
|
|
"pydanny-cookiecutter-django",
|
|
"pydron-anycall",
|
|
"pyrocat101-moo",
|
|
"python-performance",
|
|
"python-useful-helpers-advanced-descriptors",
|
|
"pytorch-vision",
|
|
"quantopian-pgcontents",
|
|
"quantum5-pyfortune",
|
|
"rackerlabs-timid",
|
|
"rainwoodman-bigfile",
|
|
"ralphbean-bugwarrior",
|
|
"raviparekh-webapp-error-handler",
|
|
"rbarrois-aionotify",
|
|
"reflect-reflect-python",
|
|
"regardscitoyens-cpc-api",
|
|
"regebro-pyroma",
|
|
"reingart-gui2py",
|
|
"respondcreate-django-versatileimagefield",
|
|
"ribalba-markdown.highlight",
|
|
"ricobl-django-importer",
|
|
"riffm-mint",
|
|
"rochacon-punch",
|
|
"rochapps-django-secure-input",
|
|
"rocky-pycolumnize",
|
|
"rocky-python3-trepan",
|
|
"rodluger-pysyzygy",
|
|
"rollbar-pyrollbar",
|
|
"rsennrich-Bleualign",
|
|
"rupertford-melody",
|
|
"rwl-godot",
|
|
"sagemath-sage-package",
|
|
"salsita-flask-raml",
|
|
"savoirfairelinux-num2words",
|
|
"saxix-sample-data-utils",
|
|
"sbneto-s3conf",
|
|
"scheibler-khard",
|
|
"schul-cloud-resources-api-v1",
|
|
"scrapinghub-dateparser",
|
|
"seebass-django-tooling",
|
|
"seiferma-deterministic_encryption_utils",
|
|
"seporaitis-mysqlparse",
|
|
"sergedomk-fiql_parser",
|
|
"sergei-maertens-django-systemjs",
|
|
"shanbay-sea",
|
|
"shaunduncan-giphypop",
|
|
"shopkick-flawless",
|
|
"siddhantgoel-tornado-sqlalchemy",
|
|
"sigmaris-python-gssapi",
|
|
"simion-pip-upgrader",
|
|
"singularityhub-sregistry-cli",
|
|
"six8-polydatum",
|
|
"skioo-django-datatrans-gateway",
|
|
"skorokithakis-django-loginas",
|
|
"slacy-minimongo",
|
|
"sloria-aiohttp_utils",
|
|
"snare-scruffy",
|
|
"snjoetw-py-synology",
|
|
"snowplow-snowplow-python-analytics-sdk",
|
|
"soimort-you-get",
|
|
"solvebio-solvebio-python",
|
|
"sontek-pyramid_webassets",
|
|
"sophilabs-django-qurl-templatetag",
|
|
"sourcesimian-pyTelegramBotAPI",
|
|
"sp4ke-howto",
|
|
"spotify-pyschema",
|
|
"srevenant-onetimejwt",
|
|
"ssherar-hook",
|
|
"stevelittlefish-littlefish",
|
|
"stovorov-IteratorDecorator",
|
|
"streamlink-streamlink",
|
|
"sublee-zeronimo",
|
|
"sundarnagarajan-cffi_utils",
|
|
"sunshowers-ntfs",
|
|
"susam-ice",
|
|
"sveetch-crispy-forms-foundation",
|
|
"swarmer-fridge",
|
|
"synw-gencharts",
|
|
"takluyver-entrypoints",
|
|
"tallero-trovotutto",
|
|
"tarmstrong-git2json",
|
|
"techdragon-python-check-pypi-name",
|
|
"tek-amino",
|
|
"tek-ribosome",
|
|
"tell-k-goolabs",
|
|
"tensorflow-probability",
|
|
"theduke-django-baseline",
|
|
"thombashi-sqliteschema",
|
|
"thusoy-pwm",
|
|
"tigrawap-pybabel-hbs",
|
|
"timheap-django-enumchoicefield",
|
|
"tjkessler-PyGenetics",
|
|
"tmontaigu-pylas",
|
|
"tnkteja-myhelp",
|
|
"tomMoral-loky",
|
|
"tomasbasham-dominos",
|
|
"tomatohater-django-unfriendly",
|
|
"tomekwojcik-ForgeryPy",
|
|
"tonyseek-flask-navigation",
|
|
"trapeze-fancy_tag",
|
|
"trec-kba-streamcorpus-pipeline",
|
|
"treycucco-pyebnf",
|
|
"trp07-messages",
|
|
"twaddington-django-gravatar",
|
|
"twz915-excel",
|
|
"ubyssey-dispatch",
|
|
"ui-django-thumbnails",
|
|
"umich-brcf-bioinf-Jacquard",
|
|
"un33k-django-toolware",
|
|
"unistra-django-rest-framework-custom-exceptions",
|
|
"uogbuji-amara3-xml",
|
|
"uploadcare-pyuploadcare",
|
|
"urda-nistbeacon",
|
|
"urinieto-msaf",
|
|
"uucidl-uu.xunitgen",
|
|
"uw-it-aca-uw-restclients-canvas",
|
|
"uw-it-aca-uw-restclients-gws",
|
|
"vaexio-vaex",
|
|
"vahtras-findifftool",
|
|
"valentinalexeev-pwaqi",
|
|
"valsaven-md5hash",
|
|
"varikin-Tigre",
|
|
"versae-ipython-cypher",
|
|
"victor-o-silva-db_file_storage",
|
|
"vint21h-nagios-check-supervisord",
|
|
"vint21h-nagios-notification-google-calendar",
|
|
"vkorn-pyvizio",
|
|
"voidfiles-lark",
|
|
"voidpp-vcp",
|
|
"vpelletier-python-libaio",
|
|
"wagtail-django-modelcluster",
|
|
"wangwenpei-fantasy",
|
|
"wdecoster-nanogui",
|
|
"wildfish-argparsetree",
|
|
"williamFalcon-test-tube",
|
|
"williballenthin-ida-settings",
|
|
"willkg-markus",
|
|
"willthames-ansible-review",
|
|
"wimglenn-wimpy",
|
|
"wreckage-django-happenings",
|
|
"wzpan-MusicBoxApi",
|
|
"xmartlabs-benderthon",
|
|
"xmikos-soapy_power",
|
|
"xtuml-pyxtuml",
|
|
"xvzf-asyncserial-py",
|
|
"yandex-yandex-tank",
|
|
"yashaka-selene",
|
|
"ybrs-single-beat",
|
|
"yeonghoey-yhy",
|
|
"yougov-motivation",
|
|
"yougov-openpack",
|
|
"youtux-pypebbleapi",
|
|
"zartstrom-snaptime",
|
|
"ze-phyr-us-django-libretto",
|
|
"zenodo-zenodo-accessrequests",
|
|
"zinic-pynsive",
|
|
"zloidemon-aiohttp_jrpc",
|
|
"zomux-deepy",
|
|
"zorg-zorg-network-camera",
|
|
"zqfang-GSEApy",
|
|
"zsennenga-module-discovery-utils"
|
|
],
|
|
"queries": 2474,
|
|
"queries_answerable": 1237,
|
|
"queries_file": "code.jsonl",
|
|
"queries_sha256": "ea7e248386350f95132a398f3710f6ce5f09424ca614fd88c618207f667a553e",
|
|
"queries_unanswerable": 1237,
|
|
"sources": [
|
|
"code-search-net/code_search_net"
|
|
]
|
|
},
|
|
"conversation": {
|
|
"corpus_file": "corpus/conversation.jsonl",
|
|
"corpus_records": 7188,
|
|
"corpus_sha256": "9cc8263c6a6d54ddc86ff1936ca57ddfd03e58d4b6dc0013cd0c47100e95a92f",
|
|
"holders": [
|
|
"00ca467f",
|
|
"01493427",
|
|
"06878be2",
|
|
"06db6396",
|
|
"06f04340",
|
|
"07b6f563",
|
|
"0862e8bf",
|
|
"08e075c7",
|
|
"08f4fc43",
|
|
"09d032c9",
|
|
"0a34ad58",
|
|
"0a995998",
|
|
"0ddfec37_abs",
|
|
"0e5e2d1a",
|
|
"0edc2aef",
|
|
"10e09553",
|
|
"118b2229",
|
|
"15745da0_abs",
|
|
"157a136e",
|
|
"16c90bf4",
|
|
"1903aded",
|
|
"195a1a1b",
|
|
"19b5f2b3_abs",
|
|
"1a1907b4",
|
|
"1c0ddc50",
|
|
"1d4e3b97",
|
|
"1da05512",
|
|
"1de5cff2",
|
|
"2133c1b5",
|
|
"21436231",
|
|
"2311e44b",
|
|
"2698e78f",
|
|
"28dc39ac",
|
|
"29f2956b",
|
|
"2a1811e2",
|
|
"2bf43736",
|
|
"311778f1",
|
|
"32260d93",
|
|
"352ab8bd",
|
|
"35a27287",
|
|
"37d43f65",
|
|
"37f165cf",
|
|
"38146c39",
|
|
"3ba21379",
|
|
"3d86fd0a",
|
|
"4100d0a0",
|
|
"41275add",
|
|
"45dc21b6",
|
|
"46a3abf7",
|
|
"488d3006",
|
|
"4c36ccef",
|
|
"4dfccbf7",
|
|
"505af2f5",
|
|
"50635ada",
|
|
"51a45a95",
|
|
"51c32626",
|
|
"54026fce",
|
|
"561fabcd",
|
|
"577d4d32",
|
|
"57f827a0",
|
|
"58470ed2",
|
|
"5a4f22c0",
|
|
"5d3d2817",
|
|
"60472f9c",
|
|
"6071bd76",
|
|
"6456829e_abs",
|
|
"65240037",
|
|
"66f24dbb",
|
|
"6a1eabeb",
|
|
"6aeb4375_abs",
|
|
"6b168ec8",
|
|
"6b7dfb22",
|
|
"6d550036",
|
|
"6e984302",
|
|
"70b3e69b",
|
|
"71a3fd6b",
|
|
"726462e0",
|
|
"7401057b",
|
|
"75499fd8",
|
|
"75832dbd",
|
|
"75f70248",
|
|
"77eafa52",
|
|
"7a8d0b71",
|
|
"830ce83f",
|
|
"8464fc84",
|
|
"853b0a1d",
|
|
"86b68151",
|
|
"87f22b4a",
|
|
"89527b6b",
|
|
"89941a94",
|
|
"8a137a7f",
|
|
"8a2466db",
|
|
"8b9d4367",
|
|
"8e91e7d9",
|
|
"8ebdbe50",
|
|
"95228167",
|
|
"95bcc1c8",
|
|
"982b5123",
|
|
"9bbe84a2",
|
|
"a08a253f",
|
|
"a2f3aa27",
|
|
"a3045048",
|
|
"a40e080f",
|
|
"a4996e51",
|
|
"a82c026e",
|
|
"a89d7624",
|
|
"af8d2e46",
|
|
"afdc33df",
|
|
"b0479f84",
|
|
"b3c15d39",
|
|
"b46e15ee",
|
|
"b6019101",
|
|
"b759caee",
|
|
"b86304ba",
|
|
"bc149d6b",
|
|
"c6853660",
|
|
"c7cf7dfd",
|
|
"c8090214_abs",
|
|
"cc06de0d",
|
|
"cc539528",
|
|
"cf22b7bf",
|
|
"d596882b",
|
|
"d905b33f",
|
|
"db467c8c",
|
|
"e3fc4d6e",
|
|
"e4e14d04",
|
|
"e5ba910e_abs",
|
|
"e61a7584",
|
|
"e8a79c70",
|
|
"ed4ddc30",
|
|
"eeda8a6d",
|
|
"f0e564bc",
|
|
"gpt4_0b2f1d21",
|
|
"gpt4_1d80365e",
|
|
"gpt4_2312f94c",
|
|
"gpt4_2f56ae70",
|
|
"gpt4_2f8be40d",
|
|
"gpt4_468eb064",
|
|
"gpt4_4ef30696",
|
|
"gpt4_5501fe77",
|
|
"gpt4_5dcc0aab",
|
|
"gpt4_6dc9b45b",
|
|
"gpt4_78cf46a3",
|
|
"gpt4_7ddcf75f",
|
|
"gpt4_85da3956",
|
|
"gpt4_93159ced_abs",
|
|
"gpt4_a2d1d1f6",
|
|
"gpt4_b5700ca9",
|
|
"gpt4_d6585ce9",
|
|
"gpt4_e414231e"
|
|
],
|
|
"queries": 300,
|
|
"queries_answerable": 150,
|
|
"queries_file": "conversation.jsonl",
|
|
"queries_sha256": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11",
|
|
"queries_unanswerable": 150,
|
|
"sources": [
|
|
"xiaowu0162/longmemeval-cleaned"
|
|
]
|
|
},
|
|
"image": {
|
|
"corpus_file": "corpus/image.jsonl",
|
|
"corpus_records": 20643,
|
|
"corpus_sha256": "4c34a3fbeddbe050431a5e8881d3a2ea56adf6af3448912e700a6f5e919eb76d",
|
|
"holders": [
|
|
"arxivqa",
|
|
"chartqa",
|
|
"infovqa",
|
|
"mp-docvqa",
|
|
"plotqa",
|
|
"slidevqa"
|
|
],
|
|
"queries": 480,
|
|
"queries_answerable": 240,
|
|
"queries_file": "image.jsonl",
|
|
"queries_sha256": "1c74488375c7ec9f3182964d51a5e506273bfd5cf3d126725c3cc431a8333128",
|
|
"queries_unanswerable": 240,
|
|
"sources": [
|
|
"openbmb/VisRAG-Ret-Test-ArxivQA",
|
|
"openbmb/VisRAG-Ret-Test-ChartQA",
|
|
"openbmb/VisRAG-Ret-Test-InfoVQA",
|
|
"openbmb/VisRAG-Ret-Test-MP-DocVQA",
|
|
"openbmb/VisRAG-Ret-Test-PlotQA",
|
|
"openbmb/VisRAG-Ret-Test-SlideVQA"
|
|
]
|
|
},
|
|
"pdf": {
|
|
"corpus_file": "corpus/pdf.jsonl",
|
|
"corpus_records": 123,
|
|
"corpus_sha256": "68f20f0b5b203b73aa08adbb4e61a512d5c9d55f60d10ea229bbdfc3322cf713",
|
|
"holders": [
|
|
"Academic-paper",
|
|
"Administration-Industry-file",
|
|
"Brochure",
|
|
"Financial-report",
|
|
"Guidebook",
|
|
"Research-report-Introduction",
|
|
"Tutorial-Workshop"
|
|
],
|
|
"queries": 984,
|
|
"queries_answerable": 766,
|
|
"queries_file": "pdf.jsonl",
|
|
"queries_sha256": "0cec488e660bf1bfa7ccfc72dbae8b7e3005f0b6a0255028e502296d4ecd4536",
|
|
"queries_unanswerable": 218,
|
|
"sources": [
|
|
"yubo2333/MMLongBench-Doc"
|
|
]
|
|
},
|
|
"ui_screenshot": {
|
|
"corpus_file": "corpus/ui_screenshot.jsonl",
|
|
"corpus_records": 730,
|
|
"corpus_sha256": "5bb110728ba5726f918139743612fcb0ca1a5e91cca2d396aaf4d3b022336570",
|
|
"holders": [
|
|
"android",
|
|
"forum",
|
|
"gitlab",
|
|
"ios",
|
|
"macos",
|
|
"shop",
|
|
"tool",
|
|
"windows"
|
|
],
|
|
"queries": 640,
|
|
"queries_answerable": 320,
|
|
"queries_file": "ui_screenshot.jsonl",
|
|
"queries_sha256": "2269dd623e67cd851fb3f6b24a4556a8462a37db98e12ad79577c5830167c7c5",
|
|
"queries_unanswerable": 320,
|
|
"sources": [
|
|
"OS-Copilot/ScreenSpot-v2"
|
|
]
|
|
},
|
|
"video": {
|
|
"corpus_file": "corpus/video.jsonl",
|
|
"corpus_records": 129,
|
|
"corpus_sha256": "6965a1e0541922fcb5f7bf943f6e67681bf3a2681bd1f196968621e870dc61c2",
|
|
"holders": [
|
|
"Artistic-Performance",
|
|
"Film-Television",
|
|
"Knowledge",
|
|
"Life-Record",
|
|
"Multilingual",
|
|
"Sports-Competition"
|
|
],
|
|
"queries": 430,
|
|
"queries_answerable": 215,
|
|
"queries_file": "video.jsonl",
|
|
"queries_sha256": "4973231d9f16d214b14923d79a34b85c3d2b423624dea752b1ee91ba09ff778c",
|
|
"queries_unanswerable": 215,
|
|
"sources": [
|
|
"lmms-eval/Video-MME"
|
|
]
|
|
}
|
|
},
|
|
"queries": 5308,
|
|
"queries_answerable": 2928,
|
|
"queries_unanswerable": 2380
|
|
} |