595 lines
58 KiB
BibTeX
Raw Permalink Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

@article{wilkinson_fair_2016,
title = {The {FAIR} Guiding Principles for scientific data management and stewardship},
volume = {3},
rights = {2016 The Author(s)},
issn = {2052-4463},
url = {https://www.nature.com/articles/sdata201618},
doi = {10.1038/sdata.2016.18},
abstract = {There is an urgent need to improve the infrastructure supporting the reuse of scholarly data. A diverse set of stakeholders—representing academia, industry, funding agencies, and scholarly publishers—have come together to design and jointly endorse a concise and measureable set of principles that we refer to as the {FAIR} Data Principles. The intent is that these may act as a guideline for those wishing to enhance the reusability of their data holdings. Distinct from peer initiatives that focus on the human scholar, the {FAIR} Principles put specific emphasis on enhancing the ability of machines to automatically find and use the data, in addition to supporting its reuse by individuals. This Comment is the first formal publication of the {FAIR} Principles, and includes the rationale behind them, and some exemplar implementations in the community.},
pages = {160018},
number = {1},
journaltitle = {Scientific Data},
shortjournal = {Sci Data},
author = {Wilkinson, Mark D. and Swertz, Morris A. and et al.},
urldate = {2024-06-22},
date = {2016-03-15},
langid = {english},
note = {Publisher: Nature Publishing Group},
keywords = {Publication characteristics, Research data},
}
@online{pydantic,
title = {Welcome to Pydantic - Pydantic},
url = {https://docs.pydantic.dev/latest/},
urldate = {2024-07-01},
}
@article{surveytestingmethods2022,
title = {Survey of Testing Methods and Testbed Development Concerning Internet of Things},
volume = {123},
issn = {1572-834X},
url = {https://doi.org/10.1007/s11277-021-09124-5},
doi = {10.1007/s11277-021-09124-5},
abstract = {The concept of Internet of Things ({IoT}) was designed to change everyday lives of people via multiple forms of computing and easy deployment of applications. In recent years, the increasing complexity of {IoT}-ready devices and processes has led to potential risks related to system reliability. Therefore, the comprehensive testing of {IoT} technology has attracted the attention of many researchers, which promotes the extensive development of {IoT} testing methods and infrastructure. However, the current research on {IoT} testing methods and testbeds mainly focuses on specific application scenarios, lacking systematic review and analysis of many applications from different points of view. This paper systematically summarizes the latest testing methods covering different {IoT} fields and discusses the development status of the existing Internet of things testbed. Findings of this review demonstrate that {IoT} testing is moving toward larger scale and intelligent testing, and that in near future, {IoT} test architecture is set to become more standardized and universally applicable with multi-technology convergence—i.e., a combination of big data, cloud computing, and artificial intelligence—being the prime focus of {IoT} testing.},
pages = {165--194},
number = {1},
journaltitle = {Wireless Personal Communications},
shortjournal = {Wireless Pers Commun},
author = {Zhu, Shicheng and Yang, Shunkun and Gou, Xiaodong and Xu, Yang and Zhang, Tao and Wan, Yueliang},
urldate = {2024-06-30},
date = {2022-03-01},
langid = {english},
keywords = {Internet of Things, {IoT} testing, Testbed, Testing method},
}
@online{recommendedformatrsLOC,
title = {Recommended Formats Statement Datasets {\textbar} Resources (Preservation, Library of Congress)},
url = {https://www.loc.gov/preservation/resources/rfs/data.html},
abstract = {Lists technical characteristics of and metadata for datasets that best support the preservation of and long-term access to these creative works. Identifies the formats the Library of Congress prefers or finds acceptable.},
type = {web page},
urldate = {2024-06-23},
}
@online{poetry,
title = {Poetry - Python dependency management and packaging made easy},
url = {https://python-poetry.org/},
urldate = {2024-06-30},
}
@online{click,
title = {Welcome to Click — Click Documentation (8.1.x)},
url = {https://click.palletsprojects.com/en/8.1.x/},
urldate = {2024-06-30},
}
@article{pmsSpinellis2012,
title = {Package Management Systems},
volume = {29},
issn = {1937-4194},
url = {https://ieeexplore.ieee.org/abstract/document/6155145},
doi = {10.1109/MS.2012.38},
abstract = {A package management system organizes and simplifies the installation and maintenance of software by standardizing and organizing the production and consumption of software collections. As a software developer, you can benefit from package managers in two ways: through a rich and stable development environment and through friction-free reuse. Promisingly, the structure that package managers bring both to the tools we use in our development process and the libraries we reuse in our products ties nicely with the recent move emphasizing {DevOps} (development operations) as an integration between software development and {IT} operations.},
pages = {84--86},
number = {2},
journaltitle = {{IEEE} Software},
author = {Spinellis, Diomidis},
urldate = {2024-06-30},
date = {2012-03},
note = {Conference Name: {IEEE} Software},
keywords = {{DevOps}, Maintenance engineering, Product management, Software libraries, Software reusability, module dependencies, package management system, shared library, software reuse},
}
@misc{rrrr2023,
title = {Repeatability, Reproducibility, Replicability, Reusability (4R) in Journals' Policies and Software/Data Management in Scientific Publications: A Survey, Discussion, and Perspectives},
url = {http://arxiv.org/abs/2312.11028},
doi = {10.48550/arXiv.2312.11028},
shorttitle = {Repeatability, Reproducibility, Replicability, Reusability (4R) in Journals' Policies and Software/Data Management in Scientific Publications},
abstract = {With the recognized crisis of credibility in scientific research, there is a growth of reproducibility studies in computer science, and although existing surveys have reviewed reproducibility from various perspectives, especially very specific technological issues, they do not address the author-publisher relationship in the publication of reproducible computational scientific articles. This aspect requires significant attention because it is the basis for reliable research. We have found a large gap between the reproducibility-oriented practices, journal policies, recommendations, publisher artifact Description/Evaluation guidelines, submission guides, technological reproducibility evolution, and its effective adoption to contribute to tackling the crisis. We conducted a narrative survey, a comprehensive overview and discussion identifying the mutual efforts required from Authors, Journals, and Technological actors to achieve reproducibility research. The relationship between authors and scientific journals in their mutual efforts to jointly improve the reproducibility of scientific results is analyzed. Eventually, we propose recommendations for the journal policies, as well as a unified and standardized Reproducibility Guide for the submission of scientific articles for authors. The main objective of this work is to analyze the implementation and experiences of reproducibility policies, techniques and technologies, standards, methodologies, software, and data management tools required for scientific reproducible publications. Also, the benefits and drawbacks of such an adoption, as well as open challenges and promising trends, to propose possible strategies and efforts to mitigate the identified gaps. To this purpose, we analyzed 200 scientific articles, surveyed 16 Computer Science journals, and systematically classified them according to reproducibility strategies, technologies, policies, code citation, and editorial business. We conclude there is still a reproducibility gap in scientific publications, although at the same time also the opportunity to reduce this gap with the joint effort of authors, publishers, and technological providers.},
number = {{arXiv}:2312.11028},
publisher = {{arXiv}},
author = {Hernández, José Armando and Colom, Miguel},
urldate = {2024-06-30},
date = {2023-12-18},
eprinttype = {arxiv},
eprint = {2312.11028 [cs]},
keywords = {Computer Science - Software Engineering, repeatability, replicability, reproducibility, reusability},
}
@thesis{vacuumpie2023,
location = {Gjøvik},
title = {Private Information Exposed by the Use of Robot Vacuum Cleaner in Smart Environments},
abstract = {Robot vacuum cleaners are popular {IoT} devices and are deployed in all kinds of
smart environments. Integration with {IoT} systems introduce more security and
privacy issues related to the operation of these devices. Vendors have developed
smart phone applications where users can personalize cleaning or view informa-
tion about the vacuum cleaner. This increase the integration between users life
and the robot vacuum cleaner, which potentially exposes private information. In-
dustry standards include end-to-end encryption between the application, cloud
service and robot vacuum cleaner to secure the private information exchanged.
Regardless of encryption, network header metadata is still available through net-
work eavesdropping attacks. In this project we investigated the potential private
information exposed by this metadata. An Irobot Roomba i7 was deployed in two
different smart environments where passive network eavesdropping was conduc-
ted during smart feature triggering. Analysis revealed that it was possible to attrib-
ute different events triggered on the Irobot Roomba i7, only based on metadata
in the Internet traffic capture. Different signature-based detection algorithms are
proposed, with a high detection rate. Wi-Fi and Internet capturing metadata were
compared and similar patterns were identified, making the detection method ap-
plicable for Wi-Fi eavesdropping as well. This thesis covers the implementation,
capturing and analysis of network traffic and proposes event detection algorithms.},
institution = {Norwegian University of Science and Technology},
type = {Master Thgesis},
author = {Ulsmåg, Benjamin Andreas},
date = {2023-01-06},
}
@inproceedings{alyamiwifi2022,
location = {Las Vegas, {NV}, {USA}},
title = {{WiFi}-based {IoT} Devices Profiling Attack based on Eavesdropping of Encrypted {WiFi} Traffic},
isbn = {978-1-66543-161-3},
url = {https://ieeexplore.ieee.org/document/9700674/},
doi = {10.1109/CCNC49033.2022.9700674},
abstract = {Recent research has shown that in-network observers of {WiFi} communication (i.e., observers who have joined the {WiFi} network) can obtain much information regarding the types, user identities, and activities of Internet-of-Things ({IoT}) devices in the network. What has not been explored is the question of how much information can be inferred by an out-ofnetwork observer who does not have access to the {WiFi} network. This attack scenario is more realistic and much harder to defend against, thus imposes a real threat to user privacy. In this paper, we investigate privacy leakage derived from an out-of-network traffic eavesdropper on the encrypted {WiFi} traffic of popular {IoT} devices. We instrumented a testbed of 12 popular {IoT} devices and evaluated multiple machine learning methods for fingerprinting and inferring what {IoT} devices exist in a {WiFi} network. By only exploiting the {WiFi} frame header information, we have achieved 95\% accuracy in identifying the devices and often their working status. This study demonstrates that information leakage and privacy attack is a real threat for {WiFi} networks and {IoT} applications.},
eventtitle = {2022 {IEEE} 19th Annual Consumer Communications \& Networking Conference ({CCNC})},
pages = {385--392},
booktitle = {2022 {IEEE} 19th Annual Consumer Communications \& Networking Conference ({CCNC})},
publisher = {{IEEE}},
author = {Alyami, Mnassar and Alharbi, Ibrahim and Zou, Cliff and Solihin, Yan and Ackerman, Karl},
urldate = {2024-03-22},
date = {2022-01-08},
langid = {english},
}
@article{tbsmartgrid2013,
title = {Cyber-Physical Security Testbeds: Architecture, Application, and Evaluation for Smart Grid},
volume = {4},
issn = {1949-3061},
url = {https://ieeexplore.ieee.org/abstract/document/6473865},
doi = {10.1109/TSG.2012.2226919},
shorttitle = {Cyber-Physical Security Testbeds},
abstract = {The development of a smarter electric grid will depend on increased deployments of information and communication technology ({ICT}) to support novel communication and control functions. Unfortunately, this additional dependency also expands the risk from cyber attacks. Designing systems with adequate cyber security depends heavily on the availability of representative environments, such as testbeds, where current issues and future ideas can be evaluated. This paper provides an overview of a smart grid security testbed, including the set of control, communication, and physical system components required to provide an accurate cyber-physical environment. It then identifies various testbed research applications and also identifies how various components support these applications. The {PowerCyber} testbed at Iowa State University is then introduced, including the architecture, applications, and novel capabilities, such as virtualization, Real Time Digital Simulators ({RTDS}), and {ISEAGE} {WAN} emulation. Finally, several attack scenarios are evaluated using the testbed to explore cyber-physical impacts. In particular, availability and integrity attacks are demonstrated with both isolated and coordinated approaches, these attacks are then evaluated based on the physical system's voltage and rotor angle stability.},
pages = {847--855},
number = {2},
journaltitle = {{IEEE} Transactions on Smart Grid},
author = {Hahn, Adam and Ashok, Aditya and Sridhar, Siddharth and Govindarasu, Manimaran},
urldate = {2024-06-30},
date = {2013-06},
note = {Conference Name: {IEEE} Transactions on Smart Grid},
keywords = {Computer architecture, Cyber-physical systems, Protocols, Real-time systems, Security, Smart grids, Software, Substations, cyber security, ieee, iot, smart grid, testbed, testbeds},
}
@incollection{iotfundamentals,
location = {Cham},
title = {{IoT} Fundamentals: Definitions, Architectures, Challenges, and Promises},
isbn = {978-3-030-30367-9},
url = {https://doi.org/10.1007/978-3-030-30367-9_1},
abstract = {The Internet is everywhere and touched almost every corner of the globe affecting our lives in previously unimagined ways. As a living entity, the Internet is constantly evolving, and now, an era of widespread connectivity through various smart devices (i.e., things) that connect with the Internet has begun. This paradigm change is generally referred to as the Internet of Things ({IoT}). Welcoming {IoT} will bring significant benefits to economies and businesses as it enables greater innovation and productivity. On the other hand, the rapid adoption of {IoT} presents new challenges regarding connectivity, security, data processing, and scalability. Because the {IoT} world is vast and versatile, it cannot be viewed as a single technology. {IoT} looks more like an umbrella covering many protocols, technologies, and concepts that depend on specific industries. In this chapter, we will seek to look at the history of {IoT}, more clearly define it, and review its terms and concepts. We will also review vertical {IoT} markets and higher-level use cases that have successfully adopted {IoT} solutions. We will also discuss the details of the business implications, business models, and opportunities of {IoT}. Finally, the complete {IoT} stack and reference architectures from smart objects, to the networks, to the cloud, and finally the applications where information is leveraged are explained.},
pages = {3--50},
booktitle = {Intelligent Internet of Things: From Device to Fog and Cloud},
publisher = {Springer International Publishing},
author = {Firouzi, Farshad and Farahani, Bahar and Weinberger, Markus and {DePace}, Gabriel and Aliee, Fereidoon Shams},
editor = {Firouzi, Farshad and Chakrabarty, Krishnendu and Nassif, Sani},
date = {2020},
doi = {10.1007/978-3-030-30367-9_1},
}
@article{whatissmartdevice2018,
title = {What is a smart device? - a conceptualisation within the paradigm of the internet of things},
volume = {6},
issn = {2213-7459},
url = {https://doi.org/10.1186/s40327-018-0063-8},
doi = {10.1186/s40327-018-0063-8},
abstract = {The Internet of Things ({IoT}) is an interconnected network of objects which range from simple sensors to smartphones and tablets; it is a relatively novel paradigm that has been rapidly gaining ground in the scenario of modern wireless telecommunications with an expected growth of 25 to 50 billion of connected devices for 2020 Due to the recent rise of this paradigm, authors across the literature use inconsistent terms to address the devices present in the {IoT}, such as mobile device, smart device, mobile technologies or mobile smart device. Based on the existing literature, this paper chooses the term smart device as a starting point towards the development of an appropriate definition for the devices present in the {IoT}. This investigation aims at exploring the concept and main features of smart devices as well as their role in the {IoT}. This paper follows a systematic approach for reviewing compendium of literature to explore the current research in this field. It has been identified smart devices as the primary objects interconnected in the network of {IoT}, having an essential role in this paradigm. The developed concept for defining smart device is based on three main features, namely context-awareness, autonomy and device connectivity. Other features such as mobility and user-interaction were highly mentioned in the literature, but were not considered because of the nature of the {IoT} as a network mainly oriented to device-to-device connectivity whether they are mobile or not and whether they interact with people or not. What emerges from this paper is a concept which can be used to homogenise the terminology used on further research in the Field of digitalisation and smart technologies.},
pages = {3},
number = {1},
journaltitle = {Visualization in Engineering},
shortjournal = {Visualization in Engineering},
author = {Silverio-Fernández, Manuel and Renukappa, Suresh and Suresh, Subashini},
date = {2018-05-09},
}
@report{dasilvaworkflow2021,
title = {Workflows Community Summit: Bringing the Scientific Workflows Community Together},
url = {http://arxiv.org/abs/2103.09181},
shorttitle = {Workflows Community Summit},
abstract = {Scientific workflows have been used almost universally across scientific domains, and have underpinned some of the most significant discoveries of the past several decades. Many of these workflows have high computational, storage, and/or communication demands, and thus must execute on a wide range of large-scale platforms, from large clouds to upcoming exascale high-performance computing ({HPC}) platforms. These executions must be managed using some software infrastructure. Due to the popularity of workflows, workflow management systems ({WMSs}) have been developed to provide abstractions for creating and executing workflows conveniently, efficiently, and portably. While these efforts are all worthwhile, there are now hundreds of independent {WMSs}, many of which are moribund. As a result, the {WMS} landscape is segmented and presents significant barriers to entry due to the hundreds of seemingly comparable, yet incompatible, systems that exist. As a result, many teams, small and large, still elect to build their own custom workflow solution rather than adopt, or build upon, existing {WMSs}. This current state of the {WMS} landscape negatively impacts workflow users, developers, and researchers. The "Workflows Community Summit" was held online on January 13, 2021. The overarching goal of the summit was to develop a view of the state of the art and identify crucial research challenges in the workflow community. Prior to the summit, a survey sent to stakeholders in the workflow community (including both developers of {WMSs} and users of workflows) helped to identify key challenges in this community that were translated into 6 broad themes for the summit, each of them being the object of a focused discussion led by a volunteer member of the community. This report documents and organizes the wealth of information provided by the participants before, during, and after the summit.},
author = {da Silva, Rafael Ferreira and Casanova, Henri and Chard, Kyle and Laney, Dan and Ahn, Dong and Jha, Shantenu and Goble, Carole and Ramakrishnan, Lavanya and Peterson, Luc and Enders, Bjoern and Thain, Douglas and Altintas, Ilkay and Babuji, Yadu and Badia, Rosa M. and Bonazzi, Vivien and Coleman, Taina and Crusoe, Michael and Deelman, Ewa and Di Natale, Frank and Di Tommaso, Paolo and Fahringer, Thomas and Filgueira, Rosa and Fursin, Grigori and Ganose, Alex and Gruning, Bjorn and Katz, Daniel S. and Kuchar, Olga and Kupresanin, Ana and Ludascher, Bertram and Maheshwari, Ketan and Mattoso, Marta and Mehta, Kshitij and Munson, Todd and Ozik, Jonathan and Peterka, Tom and Pottier, Loic and Randles, Tim and Soiland-Reyes, Stian and Tovar, Benjamin and Turilli, Matteo and Uram, Thomas and Vahi, Karan and Wilde, Michael and Wolf, Matthew and Wozniak, Justin},
urldate = {2024-02-25},
date = {2021-03-16},
doi = {10.5281/zenodo.4606958},
eprinttype = {arxiv},
eprint = {2103.09181 [cs]},
keywords = {Computer Science - Distributed, Parallel, and Cluster Computing},
}
@inproceedings{faircsartefacts2022,
location = {New York, {NY}, {USA}},
title = {Toward findable, accessible, interoperable, and reusable cybersecurity artifacts},
isbn = {978-1-4503-9684-4},
url = {https://doi.org/10.1145/3546096.3546104},
doi = {10.1145/3546096.3546104},
series = {Cset '22},
abstract = {Researchers in experimental cybersecurity are increasingly sharing the code, data, and other artifacts associated with their studies. This trend is encouraged and rewarded by conferences and journals through practices such as artifact evaluation and badging. While these trends in sharing artifacts are promising, the cybersecurity community is still far from an ecosystem in which artifacts are {FAIR}: findable, accessible, interoperable, and reusable. The lack of established standards and best practices for sharing and reuse results in artifacts that are often difficult to find and reuse; in addition, the lack of community standards results in artifacts that may be incomplete and low-quality. In this paper we describe our experience in creating an online community hub, called {SEARCCH}, to promote the sharing and reuse of artifacts for cybersecurity research. Based on our experience, we offer lessons learned: issues that must be addressed to further promote {FAIR} principles in experimental cybersecurity.},
pages = {65--70},
booktitle = {Proceedings of the 15th workshop on cyber security experimentation and test},
publisher = {Association for Computing Machinery},
author = {Balenson, David and Benzel, Terry and Eide, Eric and Emmerich, David and Johnson, David and Mirkovic, Jelena and Tinnel, Laura},
date = {2022},
note = {Number of pages: 6
Place: Virtual, {CA}, {USA}},
keywords = {{FAIR} principles, {SEARCCH}, artifact catalog, cybersecurity artifacts, reproducibility},
}
@article{islamiot2023,
title = {Internet of Things: Device Capabilities, Architectures, Protocols, and Smart Applications in Healthcare Domain},
volume = {10},
issn = {2327-4662},
url = {https://ieeexplore.ieee.org/abstract/document/9983826/references#references},
doi = {10.1109/JIOT.2022.3228795},
shorttitle = {Internet of Things},
abstract = {Nowadays, the Internet has spread to practically every country around the world and is having unprecedented effects on peoples lives. The Internet of Things ({IoT}) is getting more popular and has a high level of interest in both practitioners and academicians in the age of wireless communication due to its diverse applications. The {IoT} is a technology that enables everyday things to become savvier, everyday computation toward becoming intellectual, and everyday communication to become a little more insightful. In this article, the most common and popular {IoT} device capabilities, architectures, and protocols are demonstrated in brief to provide a clear overview of the {IoT} technology to the researchers in this area. The common {IoT} device capabilities, including hardware (Raspberry Pi, Arduino, and {ESP}8266) and software (operating systems ({OSs}), and built-in tools) platforms are described in detail. The widely used architectures that have recently evolved and used are the three-layer architecture, service-oriented architecture, and middleware-based architecture. The popular protocols for {IoT} are demonstrated which include constrained application protocol, message queue telemetry transport, extensible messaging and presence protocol, advanced message queuing protocol, data distribution service, low power wireless personal area network, Bluetooth low energy, and {ZigBee} that are frequently utilized to develop smart {IoT} applications. Additionally, this research provides an in-depth overview of the potential healthcare applications based on {IoT} technologies in the context of addressing various healthcare concerns. Finally, this article summarizes state-of-the-art knowledge, highlights open issues and shortcomings, and provides recommendations for further studies which would be quite beneficial to anyone with a desire to work in this field and make breakthroughs to get expertise in this area.},
pages = {3611--3641},
number = {4},
journaltitle = {{IEEE} Internet of Things Journal},
author = {Islam, Md. Milon and Nooruddin, Sheikh and Karray, Fakhri and Muhammad, Ghulam},
urldate = {2024-06-30},
date = {2023-02},
note = {Conference Name: {IEEE} Internet of Things Journal},
keywords = {Communication protocol, Computer architecture, Hardware, Internet of Things, Internet of Things ({IoT}), {IoT} architecture, Medical services, Protocols, Security, Software, device capabilities, healthcare applications},
}
@article{romanfeatures2013,
title = {On the features and challenges of security and privacy in distributed internet of things},
volume = {57},
issn = {1389-1286},
url = {https://www.sciencedirect.com/science/article/pii/S1389128613000054},
doi = {10.1016/j.comnet.2012.12.018},
series = {Towards a Science of Cyber Security},
abstract = {In the Internet of Things, services can be provisioned using centralized architectures, where central entities acquire, process, and provide information. Alternatively, distributed architectures, where entities at the edge of the network exchange information and collaborate with each other in a dynamic way, can also be used. In order to understand the applicability and viability of this distributed approach, it is necessary to know its advantages and disadvantages not only in terms of features but also in terms of security and privacy challenges. The purpose of this paper is to show that the distributed approach has various challenges that need to be solved, but also various interesting properties and strengths.},
pages = {2266--2279},
number = {10},
journaltitle = {Computer Networks},
shortjournal = {Computer Networks},
author = {Roman, Rodrigo and Zhou, Jianying and Lopez, Javier},
urldate = {2024-06-30},
date = {2013-07-05},
keywords = {Distributed Architectures, Internet of Things, Security, connectivity, iot, network},
}
@inproceedings{iothome2019,
location = {Santa Clara, {CA}},
title = {All things considered: An analysis of {IoT} devices on home networks},
isbn = {978-1-939133-06-9},
url = {https://www.usenix.org/conference/usenixsecurity19/presentation/kumar-deepak},
pages = {1169--1185},
booktitle = {28th {USENIX} security symposium ({USENIX} security 19)},
publisher = {{USENIX} Association},
author = {Kumar, Deepak and Shen, Kelly and Case, Benton and Garg, Deepali and Alperovich, Galina and Kuznetsov, Dmitry and Gupta, Rajarshi and Durumeric, Zakir},
date = {2019-08},
}
@inproceedings{iotInHomes2019,
title = {All Things Considered: An Analysis of \{{IoT}\} Devices on Home Networks},
isbn = {978-1-939133-06-9},
url = {https://www.usenix.org/conference/usenixsecurity19/presentation/kumar-deepak},
shorttitle = {All Things Considered},
eventtitle = {28th {USENIX} Security Symposium ({USENIX} Security 19)},
pages = {1169--1185},
author = {Kumar, Deepak and Shen, Kelly and Case, Benton and Garg, Deepali and Alperovich, Galina and Kuznetsov, Dmitry and Gupta, Rajarshi and Durumeric, Zakir},
urldate = {2024-06-30},
date = {2019},
langid = {english},
keywords = {adoption, home, iot},
}
@inproceedings{ukil_embedded_2011,
title = {Embedded security for Internet of Things},
url = {https://ieeexplore.ieee.org/abstract/document/5751382},
doi = {10.1109/NCETACS.2011.5751382},
abstract = {Internet of Things ({IoT}) consists of several tiny devices connected together to form a collaborative computing environment. {IoT} imposes peculiar constraints in terms of connectivity, computational power and energy budget, which make it significantly different from those contemplated by the canonical doctrine of security in distributed systems. In order to circumvent the problem of security in {IoT} domain, networks and devices need to be secured. In this paper, we consider the embedded device security only, assuming that network security is properly in place. It can be noticed that the existence of tiny computing devices that form ubiquity in {IoT} domain are very much vulnerable to different security attacks. In this work, we provide the requirements of embedded security, the solutions to resists different attacks and the technology for resisting temper proofing of the embedded devices by the concept of trusted computing. Our paper attempts to address the issue of security for data at rest. Addressing this issue is equivalent to addressing the security issue of the hardware platform. Our work also partially helps in addressing securing data in transit.},
eventtitle = {2011 2nd National Conference on Emerging Trends and Applications in Computer Science},
pages = {1--6},
booktitle = {2011 2nd National Conference on Emerging Trends and Applications in Computer Science},
author = {Ukil, Arijit and Sen, Jaydip and Koilakonda, Sripad},
urldate = {2024-06-30},
date = {2011-03},
keywords = {{ARM}, Computer architecture, Embedded systems, Hardware, Internet of things ({IoT}), Protocols, Security, Smart phones, Trustzone, confidentiality, embedded device, security, ubiquitous computing},
}
@online{tcpdump,
title = {Home {\textbar} {TCPDUMP} \& {LIBPCAP}},
url = {https://www.tcpdump.org/},
urldate = {2024-06-30},
}
@online{wiresharkorg,
title = {Wireshark · Go Deep},
url = {https://www.wireshark.org/},
urldate = {2024-06-30},
}
@online{pythonorg,
title = {Welcome to Python.org},
url = {https://www.python.org/},
abstract = {The official home of the Python Programming Language},
titleaddon = {Python.org},
urldate = {2024-06-30},
date = {2024-06-27},
langid = {english},
keywords = {tool},
}
@online{mitmproxy,
title = {mitmproxy - an interactive {HTTPS} proxy},
url = {https://mitmproxy.org/},
urldate = {2024-06-30},
keywords = {proxy, sniffing, tools},
}
@inproceedings{dasilvaComRoad2021,
title = {A Community Roadmap for Scientific Workflows Research and Development},
url = {https://ieeexplore.ieee.org/abstract/document/9652570/authors#authors},
doi = {10.1109/WORKS54523.2021.00016},
abstract = {The landscape of workflow systems for scientific applications is notoriously convoluted with hundreds of seemingly equivalent workflow systems, many isolated research claims, and a steep learning curve. To address some of these challenges and lay the groundwork for transforming workflows research and development, the {WorkflowsRI} and {ExaWorks} projects partnered to bring the international workflows community together. This paper reports on discussions and findings from two virtual “Workflows Community Summits” (January and April, 2021). The overarching goals of these workshops were to develop a view of the state of the art, identify crucial research challenges in the workflows community, articulate a vision for potential community efforts, and discuss technical approaches for realizing this vision. To this end, participants identified six broad themes: {FAIR} computational workflows; {AI} workflows; exascale challenges; {APIs}, interoperability, reuse, and standards; training and education; and building a workflows community. We summarize discussions and recommendations for each of these themes.},
eventtitle = {2021 {IEEE} Workshop on Workflows in Support of Large-Scale Science ({WORKS})},
pages = {81--90},
booktitle = {2021 {IEEE} Workshop on Workflows in Support of Large-Scale Science ({WORKS})},
author = {da Silva, Rafael Ferreira and Casanova, Henri and Chard, Kyle and Altintas, Ilkay and Badia, Rosa M and Balis, Bartosz and Coleman, Tainã and Coppens, Frederik and Di Natale, Frank and Enders, Bjoern and Fahringer, Thomas and Filgueira, Rosa and Fursin, Grigori and Garijo, Daniel and Goble, Carole and Howell, Dorran and Jha, Shantenu and Katz, Daniel S. and Laney, Daniel and Leser, Ulf and Malawski, Maciej and Mehta, Kshitij and Pottier, Loïc and Ozik, Jonathan and Peterson, J. Luc and Ramakrishnan, Lavanya and Soiland-Reyes, Stian and Thain, Douglas and Wolf, Matthew},
urldate = {2024-06-30},
date = {2021-11},
keywords = {{AI} workflows, Artificial intelligence, Buildings, Conferences, Research and development, Scientific workflows, Stakeholders, Standards, Training, community roadmap, data management, exascale computing, interoperability},
}
@article{zander2014survey,
title = {A survey of testbeds and experimental research infrastructures for wireless networks},
volume = {15},
pages = {1231--1246},
number = {4},
journaltitle = {{IEEE} Communications Surveys \& Tutorials},
author = {Zander, Justus and Zinner, Thomas and Bifulco, Roberto and Carle, Georg},
date = {2014},
note = {Publisher: {IEEE}},
keywords = {iot, springer, survey, testbed},
}
@article{al-hawawreh_developing_2021,
title = {Developing a Security Testbed for Industrial Internet of Things},
volume = {8},
issn = {2327-4662},
url = {https://ieeexplore.ieee.org/abstract/document/9233425},
doi = {10.1109/JIOT.2020.3032093},
abstract = {While achieving security for Industrial Internet of Things ({IIoT}) is a critical and nontrivial task, more attention is required for brownfield {IIoT} systems. This is a consequence of long life cycles of their legacy devices which were initially designed without considering security and {IoT} connectivity, but they are now becoming more connected and integrated with emerging {IoT} technologies and messaging communication protocols. Deploying today's methodologies and solutions in brownfield {IIoT} systems is not viable, as security solutions must co-exist and fit these systems' requirements. This necessitates a realistic standardized {IIoT} testbed that can be used as an optimal format to measure the credibility of security solutions of {IIoT} networks, analyze {IIoT} attack landscapes and extract threat intelligence. Developing a testbed for brownfield {IIoT} systems is considered a significant challenge as these systems are comprised of legacy, heterogeneous devices, communication layers and applications that need to be implemented holistically to achieve high fidelity. In this article, we propose a new generic end-to-end {IIoT} security testbed, with a particular focus on the brownfield system and provide details of the testbed's architectural design and the implementation process. The proposed testbed can be easily reproduced and reconfigured to support the testing activities of new processes and various security scenarios. The proposed testbed operation is demonstrated on different connected devices, communication protocols and applications. The experiments demonstrate that this testbed is effective in terms of its operation and security testing. A comparison with existing testbeds, including a table of features is provided.},
pages = {5558--5573},
number = {7},
journaltitle = {{IEEE} Internet of Things Journal},
author = {Al-Hawawreh, Muna and Sitnikova, Elena},
urldate = {2024-06-30},
date = {2021-04},
note = {Conference Name: {IEEE} Internet of Things Journal},
keywords = {Brownfield, Industrial Internet of Things ({IIoT}), Protocols, Resilience, Security, Sensors, Testing, ieee, iot, security testing, testbed},
}
@article{sibonitestbed2019,
title = {Security Testbed for Internet-of-Things Devices},
volume = {68},
issn = {1558-1721},
url = {https://ieeexplore.ieee.org/abstract/document/8565917},
doi = {10.1109/TR.2018.2864536},
abstract = {The Internet of Things ({IoT}) is a global ecosystem of information and communication technologies aimed at connecting any type of object (thing), at any time, and in any place, to each other and to the Internet. One of the major problems associated with the {IoT} is the heterogeneous nature of such deployments; this heterogeneity poses many challenges, particularly, in the areas of security and privacy. Specifically, security testing and analysis of {IoT} devices is considered a very complex task, as different security testing methodologies, including software and hardware security testing approaches, are needed. In this paper, we propose an innovative security testbed framework targeted at {IoT} devices. The security testbed is aimed at testing all types of {IoT} devices, with different software/hardware configurations, by performing standard and advanced security testing. Advanced analysis processes based on machine learning algorithms are employed in the testbed in order to monitor the overall operation of the {IoT} device under test. The architectural design of the proposed security testbed along with a detailed description of the testbed implementation is discussed. The testbed operation is demonstrated on different {IoT} devices using several specific {IoT} testing scenarios. The results obtained demonstrate that the testbed is effective at detecting vulnerabilities and compromised {IoT} devices.},
pages = {23--44},
number = {1},
journaltitle = {{IEEE} Transactions on Reliability},
author = {Siboni, Shachar and Sachidananda, Vinay and Meidan, Yair and Bohadana, Michael and Mathov, Yael and Bhairav, Suhas and Shabtai, Asaf and Elovici, Yuval},
urldate = {2024-06-30},
date = {2019-03},
note = {Conference Name: {IEEE} Transactions on Reliability},
keywords = {Hardware, Internet of Things, Internet of Things ({IoT}), {IoT} devices, Security, Software, Standards, Testing, privacy, security, testbed framework},
}
@article{fursinckorg2021,
title = {Collective knowledge: organizing research projects as a database of reusable components and portable workflows with common interfaces},
volume = {379},
url = {https://royalsocietypublishing.org/doi/full/10.1098/rsta.2020.0211},
doi = {10.1098/rsta.2020.0211},
shorttitle = {Collective knowledge},
abstract = {This article provides the motivation and overview of the Collective Knowledge Framework ({CK} or {cKnowledge}). The {CK} concept is to decompose research projects into reusable components that encapsulate research artifacts and provide unified application programming interfaces ({APIs}), command-line interfaces ({CLIs}), meta descriptions and common automation actions for related artifacts. The {CK} framework is used to organize and manage research projects as a database of such components. Inspired by the {USB} plug and play approach for hardware, {CK} also helps to assemble portable workflows that can automatically plug in compatible components from different users and vendors (models, datasets, frameworks, compilers, tools). Such workflows can build and run algorithms on different platforms and environments in a unified way using the customizable {CK} program pipeline with software detection plugins and the automatic installation of missing packages. This article presents a number of industrial projects in which the modular {CK} approach was successfully validated in order to automate benchmarking, auto-tuning and co-design of efficient software and hardware for machine learning and artificial intelligence in terms of speed, accuracy, energy, size and various costs. The {CK} framework also helped to automate the artifact evaluation process at several computer science conferences as well as to make it easier to reproduce, compare and reuse research techniques from published papers, deploy them in production, and automatically adapt them to continuously changing datasets, models and systems. The long-term goal is to accelerate innovation by connecting researchers and practitioners to share and reuse all their knowledge, best practices, artifacts, workflows and experimental results in a common, portable and reproducible format at {cKnowledge}.io.
This article is part of the theme issue Reliability and reproducibility in computational science: implementing verification, validation and uncertainty quantification in silico.},
pages = {20200211},
number = {2197},
journaltitle = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},
author = {Fursin, Grigori},
urldate = {2024-06-30},
date = {2021-03-29},
note = {Publisher: Royal Society},
keywords = {{DevOps}, {FAIR} principles, portability, reproducibility, research automation, reusability},
}
@inproceedings{friesssniffing2018,
title = {Multichannel-Sniffing-System for Real-World Analysing of Wi-Fi-Packets},
url = {https://ieeexplore.ieee.org/abstract/document/8436715},
doi = {10.1109/ICUFN.2018.8436715},
abstract = {Wireless technologies like Wi-Fi send their data using multiple channels. To analyze an environment and all Wi-Fi packets inside, a sniffing system is needed, which can sniff on all used channels of the wireless technology at the same time. This allows catching most packets on each channel. In this paper, a way to build up a multi-channel-sniffing-system ({MCSS}) is described. The test system uses several single board computers ({SBC}) with an external Wi-Fi adapter ({USB}), 19 {SBCs} are sniffing nodes ({SFN}) and one {SBC} as sending node ({SN}). The sniffing {SBCs} are placed in a cycle around the sender so that every node has the same chance to receive the simulated packets from the {SN}. For the control of all 20 {SBCs}, a self-developed software is used, which connects from the host to the clients and is used for configuring the experiments. The configuration is sent to each client and will initiate their start, so that their times are also synchronized, for this all clients are synchronised using a time server.},
eventtitle = {2018 Tenth International Conference on Ubiquitous and Future Networks ({ICUFN})},
pages = {358--364},
booktitle = {2018 Tenth International Conference on Ubiquitous and Future Networks ({ICUFN})},
author = {Friess, Kristof},
urldate = {2024-04-06},
date = {2018-07},
note = {{ISSN}: 2165-8536},
keywords = {Bluetooth, Europe, Hardware, Monitoring, Universal Serial Bus, Wireless communication, Wireless fidelity, multichannel, node.js, sbc, sniffing, wifi},
}
@inproceedings{peekaboo2020,
title = {Peek-a-Boo: I see your smart home activities, even encrypted!},
url = {http://arxiv.org/abs/1808.02741},
doi = {10.1145/3395351.3399421},
shorttitle = {Peek-a-Boo},
abstract = {A myriad of {IoT} devices such as bulbs, switches, speakers in a smart home environment allow users to easily control the physical world around them and facilitate their living styles through the sensors already embedded in these devices. Sensor data contains a lot of sensitive information about the user and devices. However, an attacker inside or near a smart home environment can potentially exploit the innate wireless medium used by these devices to exfiltrate sensitive information from the encrypted payload (i.e., sensor data) about the users and their activities, invading user privacy. With this in mind,in this work, we introduce a novel multi-stage privacy attack against user privacy in a smart environment. It is realized utilizing state-of-the-art machine-learning approaches for detecting and identifying the types of {IoT} devices, their states, and ongoing user activities in a cascading style by only passively sniffing the network traffic from smart home devices and sensors. The attack effectively works on both encrypted and unencrypted communications. We evaluate the efficiency of the attack with real measurements from an extensive set of popular off-the-shelf smart home {IoT} devices utilizing a set of diverse network protocols like {WiFi}, {ZigBee}, and {BLE}. Our results show that an adversary passively sniffing the traffic can achieve very high accuracy (above 90\%) in identifying the state and actions of targeted smart home devices and their users. To protect against this privacy leakage, we also propose a countermeasure based on generating spoofed traffic to hide the device states and demonstrate that it provides better protection than existing solutions.},
pages = {207--218},
booktitle = {Proceedings of the 13th {ACM} Conference on Security and Privacy in Wireless and Mobile Networks},
author = {Acar, Abbas and Fereidooni, Hossein and Abera, Tigist and Sikder, Amit Kumar and Miettinen, Markus and Aksu, Hidayet and Conti, Mauro and Sadeghi, Ahmad-Reza and Uluagac, Selcuk},
urldate = {2024-02-25},
date = {2020-07-08},
eprinttype = {arxiv},
eprint = {1808.02741 [cs]},
keywords = {{BLE}, Computer Science - Cryptography and Security, {ZigBee}, network traffic, privacy, smart-home, wifi},
}
@article{abuwaragaTestbed2020,
title = {Design and implementation of automated {IoT} security testbed},
volume = {88},
issn = {0167-4048},
url = {https://www.sciencedirect.com/science/article/pii/S0167404819301920},
doi = {10.1016/j.cose.2019.101648},
abstract = {The emergence of technology associated with the Internet of Things ({IoT}) is reshaping our lives, while simultaneously raising many issues due to their low level of security, which attackers can exploit for malicious purposes. This research paper conducts a comprehensive analysis of previous studies on {IoT} device security with a focus on the various tools used to test {IoT} devices and the vulnerabilities that were found. Additionally, the paper contains a survey of {IoT}-based security testbeds in the research literature. In this research study, we introduce an open source platform for identifying weaknesses in {IoT} networks and communications. The platform is easily modifiable and extendible to enable the addition of new security assessment tests and functionalities. It automates security evaluation, allowing for testing without human intervention. The testbed reports the security problems of the tested devices and can detect all attacks made against the devices. It is also designed to monitor communications within the testbed and with connected devices, enabling the system to abort if malicious activity is detected. To demonstrate the capabilities of the proposed {IoT} security testbed, it is used to examine the vulnerabilities of two {IoT} devices: a wireless camera and a smart bulb.},
pages = {101648},
journaltitle = {Computers \& Security},
shortjournal = {Computers \& Security},
author = {Abu Waraga, Omnia and Bettayeb, Meriem and Nasir, Qassim and Abu Talib, Manar},
urldate = {2024-06-30},
date = {2020-01-01},
keywords = {Automated testbed architecture, Internet of Things, {IoT} testbed, Vulnerability assessment},
}
@article{vasserman_vampire_2013,
title = {Vampire Attacks: Draining Life from Wireless Ad Hoc Sensor Networks},
volume = {12},
issn = {1558-0660},
url = {https://ieeexplore.ieee.org/document/6112758},
doi = {10.1109/TMC.2011.274},
shorttitle = {Vampire Attacks},
abstract = {Ad hoc low-power wireless networks are an exciting research direction in sensing and pervasive computing. Prior security work in this area has focused primarily on denial of communication at the routing or medium access control levels. This paper explores resource depletion attacks at the routing protocol layer, which permanently disable networks by quickly draining nodes' battery power. These "Vampire” attacks are not specific to any specific protocol, but rather rely on the properties of many popular classes of routing protocols. We find that all examined protocols are susceptible to Vampire attacks, which are devastating, difficult to detect, and are easy to carry out using as few as one malicious insider sending only protocol-compliant messages. In the worst case, a single Vampire can increase network-wide energy usage by a factor of O(N), where N in the number of network nodes. We discuss methods to mitigate these types of attacks, including a new proof-of-concept protocol that provably bounds the damage caused by Vampires during the packet forwarding phase.},
pages = {318--332},
number = {2},
journaltitle = {{IEEE} Transactions on Mobile Computing},
author = {Vasserman, Eugene Y. and Hopper, Nicholas},
urldate = {2024-06-22},
date = {2013-02},
note = {Conference Name: {IEEE} Transactions on Mobile Computing},
keywords = {Ad hoc networks, Denial of service, Energy consumption, Network topology, Routing, Routing protocols, Topology, ad hoc networks, routing, security, sensor networks, wireless networks},
}
@article{bashir2017internet,
title = {The Internet of Things testbed: a survey and evaluation},
volume = {78},
pages = {409--421},
journaltitle = {Future Generation Computer Systems},
author = {Bashir, Abid H and Gill, Khurram},
date = {2017},
note = {Publisher: Elsevier},
}
@article{vaughan2005use,
title = {The use of climate chambers in biological research},
volume = {39},
pages = {5121--5127},
number = {14},
journaltitle = {Environmental Science \& Technology},
author = {Vaughan, {TL} and Battle, {SC} and Walker, {KL}},
date = {2005},
note = {Publisher: {ACS} Publications},
}
@article{huang2011testbed,
title = {Testbed for evaluating performance of health monitoring systems},
volume = {60},
pages = {114--123},
number = {1},
journaltitle = {{IEEE} Transactions on Instrumentation and Measurement},
author = {Huang, Qinfen and Liu, Min and Garcia, Alfredo and Reynolds, Matthew},
date = {2011},
note = {Publisher: {IEEE}},
}
@online{noauthor_fhs_nodate,
title = {{FHS} Referenced Specifications},
url = {https://refspecs.linuxfoundation.org/fhs.shtml},
urldate = {2024-06-22},
}
@misc{fsh-home,
title = {3.8. /home : User home directories (optional)},
url = {https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html},
urldate = {2024-06-22},
}
@online{go-fair,
title = {{FAIR} Principles},
url = {https://www.go-fair.org/fair-principles/},
abstract = {In 2016, the {FAIR} Guiding Principles for scientific data management and stewardship were published in Scientific Data. The authors intended to provide guidelines to improve the Findability, Accessibility, Interoperability, and Reuse of digital assets. The principles emphasise machine-actionability (i.e., the capacity of… Continue reading →},
titleaddon = {{GO} {FAIR}},
urldate = {2024-06-22},
langid = {american},
}
@online{coryefelle_correcting_2016,
title = {Correcting the {IoT} History},
url = {http://www.chetansharma.com/correcting-the-iot-history/},
abstract = {In the last 5 years, {IoT} has entered the industry consciousness. There are varying forecasts calling for tremendous growth and … Continued},
titleaddon = {Chetan Sharma},
author = {{CoryEfelle}},
urldate = {2024-06-20},
date = {2016-03-14},
langid = {american},
}
@misc{noauthor_overview_2012,
title = {Overview of the Internet of things},
url = {https://handle.itu.int/11.1002/1000/11559},
shorttitle = {Y.{IoT}-overview},
abstract = {Recommendation {ITU}-T Y.2060 provides an overview of the Internet of things ({IoT}). It clarifies the concept and scope of the {IoT}, identifies the fundamental characteristics and high-level requirements of the {IoT} and describes the {IoT} reference model. The ecosystem and business models are also provided in an informative appendix.
Former {ITU}-T Y.2060 renumbered as {ITU}-T Y.4000 on 2016-02-05 without further modification and without being republished.},
number = {{ITU}-T Y.4000},
date = {2012-06-15},
}
@online{testbedOxford,
title = {test bed noun - Definition, pictures, pronunciation and usage notes {\textbar} Oxford Advanced Learner's Dictionary at {OxfordLearnersDictionaries}.com},
url = {https://www.oxfordlearnersdictionaries.com/definition/english/test-bed},
urldate = {2024-06-20},
}
@inproceedings{infoexpiot,
location = {New York, {NY}, {USA}},
title = {Information Exposure From Consumer {IoT} Devices: A Multidimensional, Network-Informed Measurement Approach},
isbn = {978-1-4503-6948-0},
url = {https://dl.acm.org/doi/10.1145/3355369.3355577},
doi = {10.1145/3355369.3355577},
series = {{IMC} '19},
shorttitle = {Information Exposure From Consumer {IoT} Devices},
abstract = {Internet of Things ({IoT}) devices are increasingly found in everyday homes, providing useful functionality for devices such as {TVs}, smart speakers, and video doorbells. Along with their benefits come potential privacy risks, since these devices can communicate information about their users to other parties over the Internet. However, understanding these risks in depth and at scale is difficult due to heterogeneity in devices' user interfaces, protocols, and functionality. In this work, we conduct a multidimensional analysis of information exposure from 81 devices located in labs in the {US} and {UK}. Through a total of 34,586 rigorous automated and manual controlled experiments, we characterize information exposure in terms of destinations of Internet traffic, whether the contents of communication are protected by encryption, what are the {IoT}-device interactions that can be inferred from such content, and whether there are unexpected exposures of private and/or sensitive information (e.g., video surreptitiously transmitted by a recording device). We highlight regional differences between these results, potentially due to different privacy regulations in the {US} and {UK}. Last, we compare our controlled experiments with data gathered from an in situ user study comprising 36 participants.},
pages = {267--279},
booktitle = {Proceedings of the Internet Measurement Conference},
publisher = {Association for Computing Machinery},
author = {Ren, Jingjing and Dubois, Daniel J. and Choffnes, David and Mandalari, Anna Maria and Kolcun, Roman and Haddadi, Hamed},
urldate = {2024-02-25},
date = {2019-10-21},
}
@inproceedings{aysom23,
title = {Are You Spying on Me? \{Large-Scale\} Analysis on \{{IoT}\} Data Exposure through Companion Apps},
isbn = {978-1-939133-37-3},
url = {https://www.usenix.org/conference/usenixsecurity23/presentation/nan},
shorttitle = {Are You Spying on Me?},
eventtitle = {32nd {USENIX} Security Symposium ({USENIX} Security 23)},
pages = {6665--6682},
author = {Nan, Yuhong and Wang, Xueqiang and Xing, Luyi and Liao, Xiaojing and Wu, Ruoyu and Wu, Jianliang and Zhang, Yifan and Wang, {XiaoFeng}},
urldate = {2024-02-25},
date = {2023},
langid = {english},
}
@unpublished{noauthor_toward_2023,
title = {Toward a common language to facilitate reproducible research and technology transfer: challenges and solutions},
url = {https://zenodo.org/records/8105339},
shorttitle = {Toward a common language to facilitate reproducible research and technology transfer},
abstract = {The keynote presentation from the 1st {ACM} conference on reproducibility and replicability ({ACM} {REP}'23).The video of this presentation is available at the {ACM} {YouTube} channel.Please don't hesitate to provide your feedback via the public Discord server from the {MLCommons} Task Force on Automation and Reproducibility and {GitHub} issues.[ {GitHub} project ] [ Public Collective Knowledge repository ][ Related reproducibility initiatives ] [ {cTuning}.org ] [ {cKnowledge}.org ]During the past 10 years, we have considerably improved the reproducibility of experimental results from published papers by introducing the artifact evaluation process with a unified artifact appendix and reproducibility checklists, Jupyter notebooks, containers, and Git repositories. On the other hand, our experience reproducing more than 200 papers shows that it can take weeks and months of painful and repetitive interactions between teams to reproduce artifacts. This effort includes decrypting numerous {README} files, examining ad-hoc artifacts and containers, and figuring out how to reproduce computational results. Furthermore, snapshot containers pose a challenge to optimize algorithms' performance, accuracy, power consumption and operational costs across diverse and rapidly evolving software, hardware, and data used in the real world.In this talk, I explain how our practical artifact evaluation experience and the feedback from researchers and evaluators motivated us to develop a simple, intuitive, technology agnostic, and English-like scripting language called Collective Mind ({CM}). It helps to automatically adapt any given experiment to any software, hardware, and data while automatically generating unified {README} files and synthesizing modular containers with a unified {API}. It is being developed by {MLCommons} to facilitate reproducible {AI}/{ML} Systems research and minimizing manual and repetitive benchmarking and optimization efforts, reduce time and costs for reproducible research, and simplify technology transfer to production. I also present several recent use cases of how {CM} helps {MLCommons}, the Student Cluster Competition, and artifact evaluation at {ACM}/{IEEE} conferences. I conclude with our development plans, new challenges, possible solutions, and upcoming reproducibility and optimization challenges powered by the {MLCommons} Collective Knowledge platform and {CM}: access.{cKnowledge}.org.},
urldate = {2024-02-25},
date = {2023-06-28},
doi = {10.5281/zenodo.8105339},
keywords = {artifact evaluation, artificial intelligence, automation, {cTuning}, chatgpt, cknowledge, collective knowledge, collective mind, competitions, llm, llm automation, machine learning, mlcommons, mlperf, optimization challenges, performance, replicability, reproducibility, reusability, systems},
}