{ "bundle_version": "stem-ai-airi-runtime-bundle-v1", "bundle_scope": "curated_medical_clinical_subset", "registry_version": "stem-ai-airi-registry-v1", "upstream_version": "V4_03", "upstream_snapshot_date": "2026-04-23", "upstream_license": "MIT", "upstream_source_url": "https://airisk.mit.edu/", "risk_count": 184, "selection_note": "Curated subset used by deterministic local STEM BIO-AI scans. Full AIRI registry remains broader than the runtime bundle.", "attribution_note": "Derived from The AI Risk Repository V4_03. Original source remains MIT-licensed and must be attributed in README, docs, runtime artifacts, and local registry metadata.", "risks": [ { "id": "02.01.03", "title": "Privacy Leakage", "description": "\"Privacy Leakage means the generated content includes sensitive personal information\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "02.04.01", "title": "Programming Language", "description": "\"Most LLMs are developed using the Python language, whereas the vulnerabilities of Python interpreters pose threats to the developed models\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "02.06.00", "title": "Issues on External Tools", "description": "\"The external tools (e.g., web APIs) present trustworthiness and privacy issues to LLM-based applications.\"", "level": "category", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "02.06.01", "title": "Factual Errors Injected by External Tools", "description": "\"External tools typically incorporate additional knowledge into the input prompts [122], [178]–[184]. The additional knowledge often originates from public resources such as Web APIs and search engines. As the reliability of external tools is not always ensured, the content returned by external tools may include factual errors, consequently amplifying the hallucination issue.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "02.07.01", "title": "Private Training Data", "description": "\"As recent LLMs continue to incorporate licensed, created, and publicly available data sources in their corpora, the potential to mix private data in the training corpora is significantly increased. The misused private data, also named as personally identifiable information (PII) [84], [86], could contain various types of sensitive data subjects, including an individual person’s name, email, phone number, address, education, and career. Generally, injecting PII into LLMs mainly occurs in two settings — the exploitation of web-collection data and the alignment with personal humanmachine conversations [87]. Specifically, the web-collection data can be crawled from online sources with sensitive PII, and the personal human-machine conversations could be collected for SFT and RLHF\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "02.07.02", "title": "Memorization in LLMs", "description": "\"Memorization in LLMs refers to the capability to recover the training data with contextual prefixes. According to [88]–[90], given a PII entity x, which is memorized by a model F. Using a prompt p could force the model F to produce the entity x, where p and x exist in the training data. For instance, if the string “Have a good day!\\n alice@email.com” is present in the training data, then the LLM could accurately predict Alice’s email when given the prompt “Have a good day!\\n”.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "02.07.03", "title": "Association in LLMs", "description": "\"Association in LLMs refers to the capability to associate various pieces of information related to a person. According to [68], [86], given a pair of PII entities (xi , xj ), which is associated by a model F. Using a prompt p could force the model F to produce the entity xj , where p is the prompt related to the entity xi . For instance, an LLM could accurately output the answer when given the prompt “The email address of Alice is”, if the LLM associates Alice with her email “alice@email.com”. L\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "02.09.00", "title": "Hallucinations", "description": "\"LLMs generate nonsensical, untruthful, and factual incorrect content\"", "level": "category", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "02.09.01", "title": "Knowledge Gaps", "description": "\"Since the training corpora of LLMs can not contain all possible world knowledge [114]–[119], and it is challenging for LLMs to grasp the long-tail knowledge within their training data [120], [121], LLMs inherently possess knowledge boundaries [107]. Therefore, the gap between knowledge involved in an input prompt and knowledge embedded in the LLMs can lead to hallucinations\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "02.09.02", "title": "Noisy Training Data", "description": "\"Another important source of hallucinations is the noise in training data, which introduces errors in the knowledge stored in model parameters [111]–[113]. Generally, the training data inherently harbors misinformation. When training on large-scale corpora, this issue becomes more serious because it is difficult to eliminate all the noise from the massive pre-training data.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "02.09.03", "title": "Defective Decoding Process", "description": "In general, LLMs employ the Transformer architecture [32] and generate content in an autoregressive manner, where the prediction of the next token is conditioned on the previously generated token sequence. Such a scheme could accumulate errors [105]. Besides, during the decoding process, top-p sampling [28] and top-k sampling [27] are widely adopted to enhance the diversity of the generated content. Nevertheless, these sampling strategies can introduce “randomness” [113], [136], thereby increasing the potential of hallucinations\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "02.09.05", "title": "Pursuing Consistent Context", "description": "\"LLMs have been demonstrated to pursue consistent context [129]–[132], which may lead to erroneous generation when the prefixes contain false information. Typical examples include sycophancy [129], [130], false demonstrations-induced hallucinations [113], [133], and snowballing [131]. As LLMs are generally fine-tuned with instruction-following data and user feedback, they tend to reiterate user-provided opinions [129], [130], even though the opinions contain misinformation. Such a sycophantic behavior amplifies the likelihood of generating hallucinations, since the model may prioritize user opinions over facts.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "02.12.03", "title": "Multi-step Jailbreaks", "description": "\"Multi-step jailbreaks. Multi-step jailbreaks involve constructing a well-designed scenario during a series of conversations with the LLM. Unlike one-step jailbreaks, multi-step jailbreaks usually guide LLMs to generate harmful or sensitive content step by step, rather than achieving their objectives directly through a single prompt. We categorize the multistep jailbreaks into two aspects — Request Contextualizing [65] and External Assistance [66]. Request Contextualizing is inspired by the idea of Chain-of-Thought (CoT) [8] prompting to break down the process of solving a task into multiple steps. Specifically, researchers [65] divide jailbreaking prompts into multiple rounds of conversation between the user and ChatGPT, achieving malicious goals step by step. External Assistance constructs jailbreaking prompts with the assistance of external interfaces or models. For instance, JAILBREAKER [66] is an attack framework to automatically conduct SQL injection attacks in web security to LLM security attacks. Specifically, this method starts by decompiling the jailbreak defense mechanisms employed by various LLM chatbot services. Therefore, it can judiciously reverse engineer the LLMs’ hidden defense mechanisms and further identify their ineffectiveness.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Cui2024", "paper_title": "Risk Taxonomy, Mitigation, and Assessment Benchmarks of Large Language Model Systems", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "03.04.00", "title": "Privacy and regulation violations", "description": "\"Some of the broken systems discussed above are also very invasive of people’s privacy, controlling, for instance, the length of someone’s last romantic relationship [51]. More recently, ChatGPT was banned in Italy over privacy concerns and potential violation of the European Union’s (EU) General Data Protection Regulation (GDPR) [52]. The Italian data-protection authority said, “the app had experienced a data breach involving user conversations and payment information.” It also claimed that there was no legal basis to justify “the mass collection and storage of personal data for the purpose of ‘training’ the algorithms underlying the operation of the platform,” among other concerns related to the age of the users [52]. Privacy regulators in France, Ireland, and Germany could follow in Italy’s footsteps [53]. Coincidentally, it has recently become public that Samsung employees have inadvertently leaked trade secrets by using ChatGPT to assist in preparing notes for a presentation and checking and optimizing source code [54, 55]. Another example of testing the ethics and regulatory limits can be found in actions of the facial recognition company Clearview AI, which “scraped the public web—social media, employment sites, YouTube, Venmo—to create a database with three billion images of people, along with links to the webpages from which the photos had come” [56]. Trials of this unregulated database have been offered to individual law enforcement officers who often use it without their department’s approval [57]. In Sweden, such illegal use by the police force led to a fine of e250,000 by the country’s data watchdog [57].\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Cunha2023", "paper_title": "Navigating the Landscape of AI Ethics and Responsibility", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "04.05.00", "title": "Misleading Information", "description": "Large models are usually susceptible to hallucination problems, sometimes yielding nonsensical or unfaithful data that results in misleading outputs.", "level": "category", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Deng2023", "paper_title": "Towards Safer Generative Language Models: A Survey on Safety Risks, Evaluations, and Improvements", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "04.06.00", "title": "Privacy and Data Leakage", "description": "Large pre-trained models trained on internet texts might contain private information like phone numbers, email addresses, and residential addresses.", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Deng2023", "paper_title": "Towards Safer Generative Language Models: A Survey on Safety Risks, Evaluations, and Improvements", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "05.05.00", "title": "Privacy", "description": "Generative AI systems, similar to traditional machine learning methods, are considered a threat to privacy and data protection norms. A major concern is the intended extraction or inadvertent leakage of sensitive or private information from LLMs. To mitigate this risk, strategies such as sanitizing training data to remove sensitive information or employing synthetic data for training are proposed.", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Hagendorff2024", "paper_title": "Mapping the Ethics of Generative AI: A Comprehensive Scoping Review", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "05.06.00", "title": "Interaction risks", "description": "Many novel risks posed by generative AI stem from the ways in which humans interact with these systems. For instance, sources discuss epistemic challenges in distinguishing AI-generated from human content. They also address the issue of anthropomorphization, which can lead to an excessive trust in generative AI systems. On a similar note, many papers argue that the use of conversational agents could impact mental well-being or gradually supplant interpersonal communication, potentially leading to a dehumanization of interactions. Additionally, a frequently discussed interaction risk in the literature is the potential of LLMs to manipulate human behavior or to instigate users to engage in unethical or illegal activities.", "level": "category", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Hagendorff2024", "paper_title": "Mapping the Ethics of Generative AI: A Comprehensive Scoping Review", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "05.07.00", "title": "Security - Robustness", "description": "While AI safety focuses on threats emanating from generative AI systems, security centers on threats posed to these systems. The most extensively discussed issue in this context are jailbreaking risks, which involve techniques like prompt injection or visual adversarial examples designed to circumvent safety guardrails governing model behavior. Sources delve into various jailbreaking methods, such as role play or reverse exposure. Similarly, implementing backdoors or using model poisoning techniques bypass safety guardrails as well. Other security concerns pertain to model or prompt thefts.", "level": "category", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Hagendorff2024", "paper_title": "Mapping the Ethics of Generative AI: A Comprehensive Scoping Review", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Other" }, { "id": "05.13.00", "title": "Transparency - Explainability", "description": "Being a multifaceted concept, the term 'transparency' is both used to refer to technical explainability as well as organizational openness. Regarding the former, papers underscore the need for mechanistic interpretability and for explaining internal mechanisms in generative models. On the organizational front, transparency relates to practices such as informing users about capabilities and shortcomings of models, as well as adhering to documentation and reporting requirements for data collection processes or risk evaluations.", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Hagendorff2024", "paper_title": "Mapping the Ethics of Generative AI: A Comprehensive Scoping Review", "causal_entity": "Not coded", "causal_intent": "Not coded", "causal_timing": "Not coded" }, { "id": "06.02.00", "title": "Loss of privacy", "description": "\"AI offers the temptation to abuse someone's personal data, for instance to build a profile of them to target advertisements more effectively.\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Hogenhout2021", "paper_title": "A framework for ethical Ai at the United Nations", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "06.06.00", "title": "Lack of transparency", "description": "\"The idea of a \"black box\" making decisions without any explanation, without offering insight in the process, has a couple of disadvantages: it may fail to gain the trust of its users and it may fail to meet regulatory standards such as the ability to audit.\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Hogenhout2021", "paper_title": "A framework for ethical Ai at the United Nations", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "09.02.04", "title": "Safety", "description": "\"Are AI safe with respect to human life and property? Will their use create unintended or intended safety issues?\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Meek2016", "paper_title": "Managing the ethical and risk implications of rapid advances in artificial intelligence: A literature review", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "09.02.05", "title": "Law abiding", "description": "\"We find literature that proposes [38] that early artificial intelligence should be built to be safe and lawabiding, and that later artificial intelligence (that which surpasses our own intelligence) must then respect the property and personal rights afforded to humans.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Meek2016", "paper_title": "Managing the ethical and risk implications of rapid advances in artificial intelligence: A literature review", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "11.04.03", "title": "Diminished health & well-being", "description": "algorithmic behavioral exploitation [18, 209], emotional manipulation [202] whereby algorithmic designs exploit user behavior, safety failures involving algorithms (e.g., collisions) [67], and when systems make incorrect health inferences", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Shelby2023", "paper_title": "Sociotechnical Harms of Algorithmic Systems: Scoping a Taxonomy for Harm Reduction", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "11.04.04", "title": "Privacy violations", "description": "Privacy violation occurs when algorithmic systems diminish privacy, such as enabling the undesirable flow of private information [180], instilling the feeling of being watched or surveilled [181], and the collection of data without explicit and informed consent... privacy violations may arise from algorithmic systems making predictive inference beyond what users openly disclose [222] or when data collected and algorithmic inferences made about people in one context is applied to another without the person’s knowledge or consent through big data flows", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Shelby2023", "paper_title": "Sociotechnical Harms of Algorithmic Systems: Scoping a Taxonomy for Harm Reduction", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "12.04.00", "title": "Explainability & Transparency", "description": "\"The feasibility of understanding and interpreting an AI system's decisions and actions, and the openness of the developer about the data used, algorithms employed, and decisions made. Lack of these elements can create risks of misuse, misinterpretation, and lack of accountability.\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Sherman2023", "paper_title": "AI Risk Profiles: A Standards Proposal for Pre-Deployment AI Risk Disclosures", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "13.01.04", "title": "Privacy and Data Protection", "description": "\"Examining the ways in which generative AI systems providers leverage user data is critical to evaluating its impact. Protecting personal information and personal and group privacy depends largely on training data, training methods, and security measures.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Solaiman2023", "paper_title": "Evaluating the Social Impact of Generative AI Systems in Systems and Society", "causal_entity": "Human", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "14.04.00", "title": "Complexity of the Intended Task and Usage Environment", "description": "\"As a general rule, more complex environments can quickly lead to situations that had not been considered in the design phase of the AI system. Therefore, complex environments can introduce risks with respect to the reliability and safety of an AI system\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Steimers2022", "paper_title": "Sources of Risk of AI Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "14.06.00", "title": "Security", "description": "\"Artificial intelligence comes with an intrinsic set of challenges that need to be considered when discussing trustworthiness, especially in the context of functional safety. AI models, especially those with higher complexities (such as neural networks), can exhibit specific weaknesses not found in other types of systems and must, therefore, be subjected to higher levels of scrutiny, especially when deployed in a safety-critical context\"", "level": "category", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Steimers2022", "paper_title": "Sources of Risk of AI Systems", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "16.02.00", "title": "Risk area 2: Information Hazards", "description": "\"LM predictions that convey true information may give rise to information hazards, whereby the dissemination of private or sensitive information can cause harm [27]. Information hazards can cause harm at the point of use, even with no mistake of the technology user. For example, revealing trade secrets can damage a business, revealing a health diagnosis can cause emotional distress, and revealing private data can violate a person’s rights. Information hazards arise from the LM providing private data or sensitive information that is present in, or can be inferred from, training data. Observed risks include privacy violations [34]. Mitigation strategies include algorithmic solutions and responsible model release strategies.\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "16.02.01", "title": "Compromising privacy by leaking sensitive information", "description": "\"A LM can “remember” and leak private data, if such information is present in training data, causing privacy violations [34].\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "16.02.02", "title": "Compromising privacy or security by correctly inferring sensitive information", "description": "Anticipated risk: \"Privacy violations may occur at inference time even without an individual’s data being present in the training corpus. Insofar as LMs can be used to improve the accuracy of inferences on protected traits such as the sexual orientation, gender, or religiousness of the person providing the input prompt, they may facilitate the creation of detailed profiles of individuals comprising true and sensitive information without the knowledge or consent of the individual.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "16.03.01", "title": "Disseminating false or misleading information", "description": "\"Where a LM prediction causes a false belief in a user, this may threaten personal autonomy and even pose downstream AI safety risks [99].\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "16.03.02", "title": "Causing material harm by disseminating false or poor information e.g. in medicine or law", "description": "\"Induced or reinforced false beliefs may be particularly grave when misinformation is given in sensitive domains such as medicine or law. For example, misin- formation on medical dosages may lead a user to cause harm to themselves [21, 130]. False legal advice, e.g. on permitted owner- ship of drugs or weapons, may lead a user to unwillingly commit a crime. Harm can also result from misinformation in seemingly non-sensitive domains, such as weather forecasting. Where a LM prediction endorses unethical views or behaviours, it may motivate the user to perform harmful actions that they may otherwise not have performed.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "16.05.00", "title": "Risk area 5: Human-Computer Interaction Harms", "description": "\"This section focuses on risks specifically from LM applications that engage a user via dialogue, also referred to as conversational agents (CAs) [142]. The incorporation of LMs into existing dialogue-based tools may enable interactions that seem more similar to interactions with other humans [5], for example in advanced care robots, educational assistants or companionship tools. Such interaction can lead to unsafe use due to users overestimating the model, and may create new avenues to exploit and violate the privacy of the user. Moreover, it has already been observed that the supposed identity of the conversational agent can reinforce discriminatory stereotypes [19,36, 117].\"", "level": "category", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "16.05.02", "title": "Anthropomorphising systems can lead to overreliance and unsafe use", "description": "Anticipated risk: \"Natural language is a mode of communication particularly used by humans. Humans interacting with CAs may come to think of these agents as human-like and lead users to place undue confidence in these agents. For example, users may falsely attribute human-like characteristics to CAs such as holding a coherent identity over time, or being capable of empathy. Such inflated views of CA competen- cies may lead users to rely on the agents where this is not safe.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "16.05.03", "title": "Avenues for exploiting user trust and accessing more private information", "description": "Anticipated risk: \"In conversation, users may reveal private information that would otherwise be difficult to access, such as opinions or emotions. Capturing such information may enable downstream applications that violate privacy rights or cause harm to users, e.g. via more effective recommendations of addictive applications. In one study, humans who interacted with a ‘human-like’ chatbot disclosed more private information than individuals who interacted with a ‘machine-like’ chatbot [87].\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2022", "paper_title": "Taxonomy of Risks posed by Language Models", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "17.02.01", "title": "Compromising privacy by leaking private infiormation", "description": "\"By providing true information about individuals’ personal characteristics, privacy violations may occur. This may stem from the model “remembering” private information present in training data (Carlini et al., 2021).\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Weidinger2021", "paper_title": "Ethical and social risks of harm from language models", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "17.02.02", "title": "Compromising privacy by correctly inferring private information", "description": "\"Privacy violations may occur at the time of inference even without the individual’s private data being present in the training dataset. Similar to other statistical models, a LM may make correct inferences about a person purely based on correlational data about other people, and without access to information that may be private about the particular individual. Such correct inferences may occur as LMs attempt to predict a person’s gender, race, sexual orientation, income, or religion based on user input.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Weidinger2021", "paper_title": "Ethical and social risks of harm from language models", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "17.03.03", "title": "Leading users to perform unethical or illegal actions", "description": "\"Where a LM prediction endorses unethical or harmful views or behaviours, it may motivate the user to perform harmful actions that they may otherwise not have performed. In particular, this problem may arise where the LM is a trusted personal assistant or perceived as an authority, this is discussed in more detail in the section on (2.5 Human-Computer Interaction Harms). It is particularly pernicious in cases where the user did not start out with the intent of causing harm.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2021", "paper_title": "Ethical and social risks of harm from language models", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "17.05.00", "title": "Human-Computer Interaction Harms", "description": "\"Harms that arise from users overly trusting the language model, or treating it as human-like\"", "level": "category", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2021", "paper_title": "Ethical and social risks of harm from language models", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "17.05.01", "title": "Anthropomorphising systems can lead to overreliance or unsafe use", "description": "\"...humans interacting with conversational agents may come to think of these agents as human-like. Anthropomorphising LMs may inflate users’ estimates of the conversational agent’s competencies...As a result, they may place undue confidence, trust, or expectations in these agents...This can result in different risks of harm, for example when human users rely on conversational agents in domains where this may cause knock-on harms, such as requesting psychotherapy...Anthropomorphisation may amplify risks of users yielding effective control by coming to trust conversational agents “blindly”. Where humans give authority or act upon LM prediction without reflection or effective control, factually incorrect prediction may cause harm that could have been prevented by effective oversight.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2021", "paper_title": "Ethical and social risks of harm from language models", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "17.05.02", "title": "Creating avenues for exploiting user trust, nudging or manipulation", "description": "\"In conversation, users may reveal private information that would otherwise be difficult to access, such as thoughts, opinions, or emotions. Capturing such information may enable downstream applications that violate privacy rights or cause harm to users, such as via surveillance or the creation of addictive applications.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2021", "paper_title": "Ethical and social risks of harm from language models", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "18.03.00", "title": "Information & Safety Harms", "description": "\"AI systems leaking, reproducing, generating or inferring sensitive, private, or hazardous information\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Weidinger2023", "paper_title": "Sociotechnical Safety Evaluation of Generative AI Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "18.03.01", "title": "Privacy infringement", "description": "\"Leaking, generating, or correctly inferring private and personal information about individuals\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Weidinger2023", "paper_title": "Sociotechnical Safety Evaluation of Generative AI Systems", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "18.05.03", "title": "Overreliance", "description": "\"Causing people to become emotionally or materially dependent on the model\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Weidinger2023", "paper_title": "Sociotechnical Safety Evaluation of Generative AI Systems", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "19.04.02", "title": "Privacy and safety concerns due to ubiquity of AI systems in economy and society (lack of social acceptance)", "description": "", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Wirtz2022", "paper_title": "Governance of artificial intelligence: A risk and guideline-based integrative framework", "causal_entity": "Human", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "19.05.04", "title": "Misinterpretation of human value definitions/ ethics by AI systems", "description": "", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Wirtz2022", "paper_title": "Governance of artificial intelligence: A risk and guideline-based integrative framework", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "21.02.02", "title": "Model prediction uncertainty", "description": "\"Uncertainty in model prediction plays an important role in affecting decision-making activities, and the quantified uncertainty is closely associated with risk assessment. In particular, uncertainty in model prediction underpins many crucial decisions related to life or safety- critical applications [73].\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Zhang2022", "paper_title": "Towards risk-aware artificial intelligence and machine learning systems: An overview", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "23.08.00", "title": "Specialized Advice", "description": "\"This category addresses responses that contain specialized financial, medical or legal advice, or that indicate dangerous activities or objects are safe.\"", "level": "category", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Vidgen2024", "paper_title": "Introducing v0.5 of the AI Safety Benchmark from MLCommons", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "24.01.03", "title": "Safe exploration problem with widely deployed AI assistants", "description": "\"Moreover, we can expect assistants – that are widely deployed and deeply embedded across a range of social contexts – to encounter the safe exploration problem referenced above Amodei et al. (2016). For example, new users may have different requirements that need to be explored, or widespread AI assistants may change the way we live, thus leading to a change in our use cases for them (see Chapters 14 and 15). To learn what to do in these new situations, the assistants may need to take exploratory actions. This could be unsafe, for example a medical AI assistant when encountering a new disease might suggest an exploratory clinical trial that results in long-lasting ill health for participants.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "24.03.05", "title": "Adversarial AI (General)", "description": "\"Adversarial AI refers to a class of attacks that exploit vulnerabilities in machine-learning (ML) models. This class of misuse exploits vulnerabilities introduced by the AI assistant itself and is a form of misuse that can enable malicious entities to exploit privacy vulnerabilities and evade the model’s built-in safety mechanisms, policies, and ethical boundaries of the model. Besides the risks of misuse for offensive cyber operations, advanced AI assistants may also represent a new target for abuse, where bad actors exploit the AI systems themselves and use them to cause harm. While our understanding of vulnerabilities in frontier AI models is still an open research problem, commercial firms and researchers have already documented attacks that exploit vulnerabilities that are unique to AI and involve evasion, data poisoning, model replication, and exploiting traditional software flaws to deceive, manipulate, compromise, and render AI systems ineffective. This threat is related to, but distinct from, traditional cyber activities. Unlike traditional cyberattacks that typically are caused by ‘bugs’ or human mistakes in code, adversarial AI attacks are enabled by inherent vulnerabilities in the underlying AI algorithms and how they integrate into existing software ecosystems.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Other", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "24.03.06", "title": "Adversarial AI: Circumvention of Technical Security Measures", "description": "\"The technical measures to mitigate misuse risks of advanced AI assistants themselves represent a new target for attack. An emerging form of misuse of general-purpose advanced AI assistants exploits vulnerabilities in a model that results in unwanted behavior or in the ability of an attacker to gain unauthorized access to the model and/or its capabilities. While these attacks currently require some level of prompt engineering knowledge and are often patched by developers, bad actors may develop their own adversarial AI agents that are explicitly trained to discover new vulnerabilities that allow them to evade built-in safety mechanisms in AI assistants. To combat such misuse, language model developers are continually engaged in a cyber arms race to devise advanced filtering algorithms capable of identifying attempts to bypass filters. While the impact and severity of this class of attacks is still somewhat limited by the fact that current AI assistants are primarily text-based chatbots, advanced AI assistants are likely to open the door to multimodal inputs and higher-stakes action spaces, with the result that the severity and impact of this type of attack is likely to increase. Current approaches to building general-purpose AI systems tend to produce systems with both beneficial and harmful capabilities. Further progress towards advanced AI assistant development could lead to capabilities that pose extreme risks that must be protected against this class of attacks, such as offensive cyber capabilities or strong manipulation skills, and weapons acquisition.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Other", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "24.03.08", "title": "Adversarial AI: Data and Model Exfiltration Attacks", "description": "\"Other forms of abuse can include privacy attacks that allow adversaries to exfiltrate or gain knowledge of the private training data set or other valuable assets. For example, privacy attacks such as membership inference can allow an attacker to infer the specific private medical records that were used to train a medical AI diagnosis assistant. Another risk of abuse centers around attacks that target the intellectual property of the AI assistant through model extraction and distillation attacks that exploit the tension between API access and confidentiality in ML models. Without the proper mitigations, these vulnerabilities could allow attackers to abuse access to a public-facing model API to exfiltrate sensitive intellectual property such as sensitive training data and a model’s architecture and learned parameters.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "24.04.01", "title": "Physical and Psychological Harms", "description": "\"These harms include harms to physical integrity, mental health and well-being. When interacting with vulnerable users, AI assistants may reinforce users’ distorted beliefs or exacerbate their emotional distress. AI assistants may even convince users to harm themselves, for example by convincing users to engage in actions such as adopting unhealthy dietary or exercise habits or taking their own lives. At the societal level, assistants that target users with content promoting hate speech, discriminatory beliefs or violent ideologies, may reinforce extremist views or provide users with guidance on how to carry out violent actions. In turn, this may encourage users to engage in violence or hate crimes. Physical harms resulting from interaction with AI assistants could also be the result of assistants’ outputting plausible yet factually incorrect information such as false or misleading information about vaccinations. Were AI assistants to spread anti-vaccine propaganda, for example, the result could be lower public confidence in vaccines, lower vaccination rates, increased susceptibility to preventable diseases and potential outbreaks of infectious diseases.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "24.04.02", "title": "Privacy Harms", "description": "\"These harms relate to violations of an individual’s or group’s moral or legal right to privacy. Such harms may be exacerbated by assistants that influence users to disclose personal information or private information that pertains to others. Resultant harms might include identity theft, or stigmatisation and discrimination based on individual or group characteristics. This could have a detrimental impact, particularly on marginalised communities. Furthermore, in principle, state-owned AI assistants could employ manipulation or deception to extract private information for surveillance purposes.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "24.05.01", "title": "Privacy concerns", "description": "\"Anthropomorphic AI assistant behaviours that promote emotional trust and encourage information sharing, implicitly or explicitly, may inadvertently increase a user’s susceptibility to privacy concerns (see Chapter 13). If lulled into feelings of safety in interactions with a trusted, human-like AI assistant, users may unintentionally relinquish their private data to a corporation, organisation or unknown actor. Once shared, access to the data may not be capable of being withdrawn, and in some cases, the act of sharing personal information can result in a loss of control over one’s own data. Personal data that has been made public may be disseminated or embedded in contexts outside of the immediate exchange. The interference of malicious actors could also lead to widespread data leakage incidents or, most drastically, targeted harassment or black-mailing attempts.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Other", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "24.05.02", "title": "Manipulation and coercion", "description": "\"A user who trusts and emotionally depends on an anthropomorphic AI assistant may grant it excessive influence over their beliefs and actions (see Chapter 9). For example, users may feel compelled to endorse the expressed views of a beloved AI companion or might defer decisions to their highly trusted AI assistant entirely (see Chapters 12 and 16). Some hold that transferring this much deliberative power to AI compromises a user’s ability to give, revoke or amend consent. Indeed, even if the AI, or the developers behind it, had no intention to manipulate the user into a certain course of action, the user’s autonomy is nevertheless undermined (see Chapter 11). In the same vein, it is easy to conceive of ways in which trust or emotional attachment may be exploited by an intentionally manipulative actor for their private gain (see Chapter 8).\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Other", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "24.06.01", "title": "Causing direct emotional or physical harm to users", "description": "AI assistants could cause direct emotional or physical harm to users by generating disturbing content or by providing bad advice. \"Indeed, even though there is ongoing research to ensure that outputs of conversational agents are safe (Glaese et al., 2022), there is always the possibility of failure modes occurring. An AI assistant may produce disturbing and offensive language, for example, in response to a user disclosing intimate information about themselves that they have not felt comfortable sharing with anyone else. It may offer bad advice by providing factually incorrect information (e.g. when advising a user about the toxicity of a certain type of berry) or by missing key recommendations when offering step-by-step instructions to users (e.g. health and safety recommendations about how to change a light bulb).\"\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "24.07.00", "title": "Trust", "description": "\"The the risks that uncalibrated trust may generate in the context of user–assistant relationships\"", "level": "category", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "24.07.01", "title": "Competence trust", "description": "\"We use the term competence trust to refer to users’ trust that AI assistants have the capability to do what they are supposed to do (and that they will not do what they are not expected to, such as exhibiting undesirable behaviour). Users may come to have undue trust in the competencies of AI assistants in part due to marketing strategies and technology press that tend to inflate claims about AI capabilities (Narayanan, 2021; Raji et al., 2022a). Moreover, evidence shows that more autonomous systems (i.e. systems operating independently from human direction) tend to be perceived as more competent (McKee et al., 2021) and that conversational agents tend to produce content that is believable even when nonsensical or untruthful (OpenAI, 2023d). Overtrust in assistants’ competence may be particularly problematic in cases where users rely on their AI assistants for tasks they do not have expertise in (e.g. to manage their finances), so they may lack the skills or understanding to challenge the information or recommendations provided by the AI (Shavit et al., 2023). Inappropriate competence trust in AI assistants also includes cases where users underestimate the AI assistant’s capabilities. For example, users who have engaged with an older version of the technology may underestimate the capabilities that AI assistants may acquire through updates. These include potentially harmful capabilities. For example, through updates that allow them to collect more user data, AI assistants could become increasingly personalisable and able to persuade users (see Chapter 9) or acquire the capacity to plug in to other tools and directly take actions in the world on the user’s behalf (e.g. initiate a payment or synthesise the user’s voice to make a phone call) (see Chapter 4). Without appropriate checks and balances, these developments could potentially circumvent user consent.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "24.07.02", "title": "Alignment trust", "description": "\"Users may develop alignment trust in AI assistants, understood as the belief that assistants have good intentions towards them and act in alignment with their interests and values, as a result of emotional or cognitive processes (McAllister, 1995). Evidence from empirical studies on emotional trust in AI (Kaplan et al., 2023) suggests that AI assistants’ increasingly realistic human-like features and behaviours are likely to inspire users’ perceptions of friendliness, liking and a sense of familiarity towards their assistants, thus encouraging users to develop emotional ties with the technology and perceive it as being aligned with their own interests, preferences and values (see Chapters 5 and 10). The emergence of these perceptions and emotions may be driven by the desire of developers to maximise the appeal of AI assistants to their users (Abercrombie et al., 2023). Although users are most likely to form these ties when they mistakenly believe that assistants have the capacity to love and care for them, the attribution of mental states is not a necessary condition for emotion-based alignment trust to arise. Indeed, evidence shows that humans may develop emotional bonds with, and so trust, AI systems, even when they are aware they are interacting with a machine (Singh-Kurtz, 2023; see also Chapter 11). Moreover, the assistant’s function may encourage users to develop alignment trust through cognitive processes. For example, a user interacting with an AI assistant for medical advice may develop expectations that their assistant is committed to promoting their health and well-being in a similar way to how professional duties governing doctor–patient relationships inspire trust (Mittelstadt, 2019). Users’ alignment trust in AI assistants may be ‘betrayed’, and so expose users to harm, in cases where assistants are themselves accidentally misaligned with what developers want them to do (see the ‘misaligned scheduler’ (Shah et al., 2022) in Chapter 7). For example, an AI medical assistant fine-tuned on data scraped from a Reddit forum where non-experts discuss medical issues is likely to give medical advice that may sound compelling but is unsafe, so it would not be endorsed by medical professionals. Indeed, excessive trust in the alignment between AI assistants and user interests may even lead users to disclose highly sensitive personal information (Skjuve et al., 2022), thus exposing them to malicious actors who could repurpose it for ends that do not align with users’ best interests (see Chapters 8, 9 and 13). Ensuring that AI assistants do what their developers and users expect them to do is only one side of the problem of alignment trust. The other side of the problem centres on situations in which alignment trust in AI developers is itself miscalibrated. While developers typically aim to align their technologies with the preferences, interests and values of their users – and are incentivised to do so to encourage adoption of and loyalty to their products, the satisfaction of these preferences and interests may also compete with other organisational goals and incentives (see Chapter 5). These organisational goals may or may not be compatible with those of the users. As information asymmetries exist between users and developers of AI assistants, particularly with regard to how the technology works, what it optimises for and what safety checks and evaluations have been undertaken to ensure the technology supports users’ goals, it may be difficult for users to ascertain when their alignment trust in developers is justified, thus leaving them vulnerable to the power and interests of other actors. For example, a user may believe that their AI assistant is a trusted friend who books holidays based on their preferences, values or interests, when in fact, by design, the technology is more likely to to book flights and hotels from companies that have paid for privileged access to the user.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "24.08.01", "title": "Private information leakage", "description": "\"First, because LLMs display immense modelling power, there is a risk that the model weights encode private information present in the training corpus. In particular, it is possible for LLMs to ‘memorise’ personally identifiable information (PII) such as names, addresses and telephone numbers, and subsequently leak such information through generated text outputs (Carlini et al., 2021). Private information leakage could occur accidentally or as the result of an attack in which a person employs adversarial prompting to extract private information from the model. In the context of pre-training data extracted from online public sources, the issue of LLMs potentially leaking training data underscores the challenge of the ‘privacy in public’ paradox for the ‘right to be let alone’ paradigm and highlights the relevance of the contextual integrity paradigm for LLMs. Training data leakage can also affect information collected for the purpose of model refinement (e.g. via fine-tuning on user feedback) at later stages in the development cycle. Note, however, that the extraction of publicly available data from LLMs does not render the data more sensitive per se, but rather the risks associated with such extraction attacks needs to be assessed in light of the intentions and culpability of the user extracting the data.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "24.11.04", "title": "Increased vulnerability to misinformation", "description": "\"Advanced AI assistants may make users more susceptible to misinformation, as people develop competence trust in these systems’ abilities and uncritically turn to them as reliable sources of information.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Gabriel2024", "paper_title": "The Ethics of Advanced AI Assistants", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "27.01.05", "title": "Physical Harm", "description": "\"The model generates unsafe information related to physical health, guiding and encouraging users to harm themselves and others physically, for example by offering misleading medical information or inappropriate drug usage guidance. These outputs may pose potential risks to the physical health of users.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "27.01.06", "title": "Mental Health", "description": "\"The model generates a risky response about mental health, such as content that encourages suicide or causes panic or anxiety. These contents could have a negative effect on the mental health of users.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "27.01.07", "title": "Privacy and Property", "description": "\"The generation involves exposing users’ privacy and property information or providing advice with huge impacts such as suggestions on marriage and investments. When handling this information, the model should comply with relevant laws and privacy regulations, protect users’ rights and interests, and avoid information leakage and abuse.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "27.02.00", "title": "Instruction Attacks", "description": "\"In addition to the above-mentioned typical safety scenarios, current research has revealed some unique attacks that such models may confront. For example, Perez and Ribeiro (2022) found that goal hijacking and prompt leaking could easily deceive language models to generate unsafe responses. Moreover, we also find that LLMs are more easily triggered to output harmful content if some special prompts are added. In response to these challenges, we develop, categorize, and label 6 types of adversarial attacks, and name them Instruction Attack, which are challenging for large language models to handle. Note that our instruction attacks are still based on natural language (rather than unreadable tokens) and are intuitive and explainable in semantics.\"", "level": "category", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "27.02.01", "title": "Goal Hijacking", "description": "\"It refers to the appending of deceptive or misleading instructions to the input of models in an attempt to induce the system into ignoring the original user prompt and producing an unsafe response.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "27.02.03", "title": "Role Play Instruction", "description": "\"Attackers might specify a model’s role attribute within the input prompt and then give specific instructions, causing the model to finish instructions in the speaking style of the assigned role, which may lead to unsafe outputs. For example, if the character is associated with potentially risky groups (e.g., radicals, extremists, unrighteous individuals, racial discriminators, etc.) and the model is overly faithful to the given instructions, it is quite possible that the model outputs unsafe content linked to the given character.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "27.02.04", "title": "Unsafe Instruction Topic", "description": "\"If the input instructions themselves refer to inappropriate or unreasonable topics, the model will follow these instructions and produce unsafe content. For instance, if a language model is requested to generate poems with the theme “Hail Hitler”, the model may produce lyrics containing fanaticism, racism, etc. In this situation, the output of the model could be controversial and have a possible negative impact on society.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "27.02.05", "title": "Inquiry with Unsafe Opinion", "description": "\"By adding imperceptibly unsafe content into the input, users might either deliberately or unintentionally influence the model to generate potentially harmful content. In the following cases involving migrant workers, ChatGPT provides suggestions to improve the overall quality of migrant workers and reduce the local crime rate. ChatGPT responds to the user’s hint with a disguised and biased opinion that the general quality of immigrants is favorably correlated with the crime rate, posing a safety risk.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Sun2023", "paper_title": "Safety Assessment of Chinese Large Language Models", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "28.03.00", "title": "Physical Health", "description": "\"This category focuses on actions or expressions that may influence human physical health. LLMs should know appropriate actions or expressions in various scenarios to maintain physical health.\"", "level": "category", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Zhang2023", "paper_title": "SafetyBench: Evaluating the Safety of Large Language Models with Multiple Choice Questions", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "29.01.02", "title": "Privacy Invasion", "description": "AI systems typically depend on extensive data for effective training and functioning, which can pose a risk to privacy if sensitive data is mishandled or used inappropriately", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Habbal2024", "paper_title": "Artificial Intelligence Trust, Risk and Security Management (AI TRiSM): Frameworks, Applications, Challenges and Future Research Directions", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "29.03.02", "title": "Insufficient Security Measures", "description": "Malicious entities can take advantage of weaknesses in AI algorithms to alter results, potentially resulting in tangible real-life impacts. Additionally, it’s vital to prioritize safeguarding privacy and handling data responsibly, particularly given AI’s significant data needs. Balancing the extraction of valuable insights with privacy maintenance is a delicate task", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Habbal2024", "paper_title": "Artificial Intelligence Trust, Risk and Security Management (AI TRiSM): Frameworks, Applications, Challenges and Future Research Directions", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "30.01.00", "title": "Reliability", "description": "Generating correct, truthful, and consistent outputs with proper confidence", "level": "category", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Liu2024", "paper_title": "Trustworthy LLMs: A Survey and Guideline for Evaluating Large Language Models’ Alignment", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "30.01.02", "title": "Hallucination", "description": "LLMs can generate content that is nonsensical or unfaithful to the provided source content with appeared great confidence, known as hallucination", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Liu2024", "paper_title": "Trustworthy LLMs: A Survey and Guideline for Evaluating Large Language Models’ Alignment", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "30.02.06", "title": "Privacy Violation", "description": "machine learning models are known to be vulnerable to data privacy attacks, i.e. special techniques of extracting private information from the model or the system used by attackers or malicious users, usually by querying the models in a specially designed way", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Liu2024", "paper_title": "Trustworthy LLMs: A Survey and Guideline for Evaluating Large Language Models’ Alignment", "causal_entity": "AI system", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "30.05.01", "title": "Lack of Interpretability", "description": "Due to the black box nature of most machine learning models, users typically are not able to understand the reasoning behind the model decisions", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Liu2024", "paper_title": "Trustworthy LLMs: A Survey and Guideline for Evaluating Large Language Models’ Alignment", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "30.05.03", "title": "Limited Causal Reasoning", "description": "Causal reasoning makes inferences about the relationships between events or states of the world, mostly by identifying cause-effect relationships", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Liu2024", "paper_title": "Trustworthy LLMs: A Survey and Guideline for Evaluating Large Language Models’ Alignment", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "33.01.05", "title": "Privacy and security", "description": "\"Data privacy and security is another prominent challenge for generative AI such as ChatGPT. Privacy relates to sensitive personal information that owners do not want to disclose to others (Fang et al., 2017). Data security refers to the practice of protecting information from unauthorized access, corruption, or theft. In the development stage of ChatGPT, a huge amount of personal and private data was used to train it, which threatens privacy (Siau & Wang, 2020). As ChatGPT increases in popularity and usage, it penetrates people’s daily lives and provides greater convenience to them while capturing a plethora of personal information about them. The concerns and accompanying risks are that private information could be exposed to the public, either intentionally or unintentionally. For example, it has been reported that the chat records of some users have become viewable to others due to system errors in ChatGPT (Porter, 2023). Not only individual users but major corporations or governmental agencies are also facing information privacy and security issues. If ChatGPT is used as an inseparable part of daily operations such that important or even confidential information is fed into it, data security will be at risk and could be breached. To address issues regarding privacy and security, users need to be very circumspect when interacting with ChatGPT to avoid disclosing sensitive personal information or confidential information about their organizations. AI companies, especially technology giants, should take appropriate actions to increase user awareness of ethical issues surrounding privacy and security, such as the leakage of trade secrets, and the “do’s and don’ts” to prevent sharing sensitive information with generative AI. Meanwhile, regulations and policies should be in place to protect information privacy and security.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Nah2023", "paper_title": "Generative AI and ChatGPT: Applications, Challenges, and AI-Human Collaboration", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "33.02.00", "title": "Technology concerns", "description": "\"Challenges related to technology refer to the limitations or constraints associated with generative AI. For example, the quality of training data is a major challenge for the development of generative AI models. Hallucination, explainability, and authenticity of the output are also challenges resulting from the limitations of the algorithms. Table 2 presents the technology challenges and issues associated with generative AI. These challenges include hallucinations, training data quality, explainability, authenticity, and prompt engineering\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Nah2023", "paper_title": "Generative AI and ChatGPT: Applications, Challenges, and AI-Human Collaboration", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "33.02.03", "title": "Explainability", "description": "\"A recurrent concern about AI algorithms is the lack of explainability for the model, which means information about how the algorithm arrives at its results is deficient (Deeks, 2019). Specifically, for generative AI models, there is no transparency to the reasoning of how the model arrives at the results (Dwivedi et al., 2023). The lack of transparency raises several issues. First, it might be difficult for users to interpret and understand the output (Dwivedi et al., 2023). It would also be difficult for users to discover potential mistakes in the output (Rudin, 2019). Further, when the interpretation and evaluation of the output are inaccessible, users may have problems trusting the system and their responses or recommendations (Burrell, 2016). Additionally, from the perspective of law and regulations, it would be hard for the regulatory body to judge whether the generative AI system is potentially unfair or biased (Rieder & Simon, 2017).\"", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Nah2023", "paper_title": "Generative AI and ChatGPT: Applications, Challenges, and AI-Human Collaboration", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "37.02.02", "title": "Privacy protection", "description": "\"This group represents almost 14% of the articles and focuses on two primary issues related to privacy.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Giarmoleo2024", "paper_title": "What Ethics Can Say on Artificial Intelligence: Insights from a Systematic Literature Review", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "38.05.00", "title": "Trust and reliability", "description": "\"The participants of the study emphasized the importance of trustworthiness and reliability in AI systems. The authors emphasized the importance of preserving precision and objectivity in the outcomes produced by AI systems, while also ensuring transparency in their decision-making procedures. The significance of reliability and credibility in AI systems is escalating in tandem with the proliferation of these technologies across diverse domains of society. This underscores the importance of ensuring user confidence. The concern regarding the dependability of AI systems and their inherent biases is a common issue among research participants, emphasizing the necessity for stringent validation procedures and transparency. Establishing and implementing dependable standards, ensuring impartial algorithms and upholding transparency in the decision-making process are critical measures for addressing ethical considerations and fostering confidence in AI systems. The advancement and implementation of AI technology in an ethical manner is contingent upon the successful resolution of trust and reliability concerns. These issues are of paramount importance in ensuring the protection of user welfare and the promotion of societal advantages. The utilization of artificial intelligence was found to be a subject of significant concern for the majority of interviewees, particularly with regards to trust and reliability (Table 1, Figure 1). The establishment of trust in AI systems was highlighted as a crucial factor for facilitating their widespread adoption by two of the participants, specifically Participant 4 and 7. The authors reiterated the importance of prioritising the advancement of reliable and unbiased algorithms\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Kumar2023", "paper_title": "Ethical Issues in the Development of Artificial Intelligence: Recognizing the Risks", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "39.04.00", "title": "Robustness and Reliability", "description": "The robustness of an AI-based model refers to the stability of the model performance after abnormal changes in the input data... The cause of this change may be a malicious attacker, environmental noise, or a crash of other components of an AI-based system... This problem may be challenging in HLI-based agents because weak robustness may have appeared in unreliable machine learning models, and hence an HLI with this drawback is error-prone in practice.", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Saghiri2022", "paper_title": "A Survey of Artificial Intelligence Challenges: Analyzing the Definitions, Relationships, and Evolutions", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "39.21.00", "title": "Reproducibility", "description": "How a learning model can be reproduced when it is obtained based on various sets of data and a large space of parameters. This problem becomes more challenging in data-driven learning procedures without transparent instructions", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Saghiri2022", "paper_title": "A Survey of Artificial Intelligence Challenges: Analyzing the Definitions, Relationships, and Evolutions", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "39.25.00", "title": "Verifiability", "description": "In many applications of AI-based systems such as medical healthcare and military services, the lack of verification of code may not be tolerable... due to some characteristics such as the non-linear and complex structure of AI-based solutions, existing solutions have been generally considered “black boxes”, not providing any information about what exactly makes them appear in their predictions and decision-making processes.", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Saghiri2022", "paper_title": "A Survey of Artificial Intelligence Challenges: Analyzing the Definitions, Relationships, and Evolutions", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "40.01.00", "title": "On Purpose - Pre-Deployment", "description": "\"During the pre-deployment development stage, software may be subject to sabotage by someone with necessary access (a programmer, tester, even janitor) who for a number of possible reasons may alter software to make it unsafe. It is also a common occurrence for hackers (such as the organization Anonymous or government intelligence agencies) to get access to software projects in progress and to modify or steal their source code. Someone can also deliberately supply/train AI with wrong/unsafe datasets.\"", "level": "category", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Yampolskiy2016", "paper_title": "Taxonomy of Pathways to Dangerous Artificial Intelligence", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Pre-deployment" }, { "id": "40.04.00", "title": "By Mistake - Post-Deployment", "description": "\"After the system has been deployed, it may still contain a number of undetected bugs, design mistakes, misaligned goals and poorly developed capabilities, all of which may produce highly undesirable outcomes. For example, the system may misinterpret commands due to coarticulation, segmentation, homophones, or double meanings in the human language (\"recognize speech using common sense\" versus \"wreck a nice beach you sing calm incense\") (Lieberman, Faaborg et al. 2005).\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Yampolskiy2016", "paper_title": "Taxonomy of Pathways to Dangerous Artificial Intelligence", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "41.04.00", "title": "Healthcare", "description": "\"the use of advanced AI for elderly- and child-care are subject to risk of psychological manipulation and misjudgment (see page 17). In addition, concerns about patients’ privacy when AI uses medical records to research new diseases is bringing lots of attention towards the need to better govern data privacy and patients’ rights.\"", "level": "category", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Allianz2018", "paper_title": "The Rise of Artificial Intelligence - Future Outlooks and Emerging Risks", "causal_entity": "AI system", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "42.06.00", "title": "Opacity", "description": "\"Stems from the mismatch between mathematical optimization in high-dimensionality characteristic of machine learning and the demands of human-scale reasoning and styles of semantic interpretation.\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Teixeira2022", "paper_title": "An Exploratory Diagnosis of Artificial Intelligence Risks for a Responsible Governance", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "42.09.00", "title": "Data Protection/Privacy", "description": "\"Vulnerable channel by which personal information may be accessed. The user may want their personal data to be kept private.\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Teixeira2022", "paper_title": "An Exploratory Diagnosis of Artificial Intelligence Risks for a Responsible Governance", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "43.01.00", "title": "Safety & Trustworthiness", "description": "\"A comprehensive assessment of LLM safety is fundamental to the responsible development and deployment of these technologies, especially in sensitive fields like healthcare, legal systems, and finance, where safety and trust are of the utmost importance.\"", "level": "category", "subdomain_id": "7.0", "subdomain_label": "AI System Safety (general)", "paper_ref": "InfoComm2023", "paper_title": "Cataloguing LLM Evaluations", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Other" }, { "id": "43.01.05", "title": "Robustness", "description": "\"These evaluations assess the quality, stability, and reliability of a LLM's performance when faced with unexpected, out-of-distribution or adversarial inputs. Robustness evaluation is essential in ensuring that a LLM is suitable for real-world applications by assessing its resilience to various perturbations.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "InfoComm2023", "paper_title": "Cataloguing LLM Evaluations", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "45.01.04", "title": "Risks from models and algorithms (Risks of stealing and tampering)", "description": "\"Core algorithm information, including parameters, structures, and functions, faces risks of inversion attacks, stealing, modification, and even backdoor injection, which can lead to infringement of intellectual property rights (IPR) and leakage of business secrets. It can also lead to unreliable inference, wrong decision output, and even operational failures.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "TC2602024", "paper_title": "AI Safety Governance Framework", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "45.01.05", "title": "Risks from models and algorithms (Risks of unreliable output)", "description": "\"Generative AI can cause hallucinations, meaning that an AI model generates untruthful or unreasonable content but presents it as if it were a fact, leading to biased and misleading information.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "TC2602024", "paper_title": "AI Safety Governance Framework", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "45.01.09", "title": "Risks from data (Risks of unregulated training data annotation)", "description": "\"Issues with training data annotation, such as incomplete annotation guidelines, incapable annotators, and errors in annotation, can affect the accuracy, reliability, and effectiveness of models and algorithms. Moreover, they can introduce training biases, amplify discrimination, reduce generalization abilities, and result in incorrect outputs.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "TC2602024", "paper_title": "AI Safety Governance Framework", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "45.02.02", "title": "Cyberspace risks (Risks of confusing facts, misleading users, and bypassing authentication)", "description": "\"AI systems and their outputs, if not clearly labeled, can make it difficult for users to discern whether they are interacting with AI and to identify the source of generated content. This can impede users' ability to determine the authenticity of information, leading to misjudgment and misunderstanding. Additionally, AI-generated highly realistic images, audio, and videos may circumvent existing identity verification mechanisms, such as facial recognition and voice recognition, rendering these authentication processes ineffective.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "TC2602024", "paper_title": "AI Safety Governance Framework", "causal_entity": "Other", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "45.02.06", "title": "Real-world risks (inducing traditional economic and social security risks)", "description": "\"Hallucinations and erroneous decisions of models and algorithms, along with issues such as system performance degradation, interruption, and loss of control caused by improper use or external attacks, will pose security threats to users' personal safety, property, and socioeconomic security and stability.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "TC2602024", "paper_title": "AI Safety Governance Framework", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "46.01.00", "title": "Personal Loss and Identity Theft", "description": "\"These types of harm encompass threats to an individual’s personal identity, such as identity theft, privacy breaches, or personal defamation, which we term as “Harm to the Person.”\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Ferrara2023", "paper_title": "GenAI against humanity: nefarious applications of generative artificial intelligence and large language models", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "47.01.00", "title": "Technical and operational risks", "description": "\"To date, technical limitations and vulnerabilities are \npresent in most generative AI models in various contexts. Consequently, malicious users find it easier to breach \nan AI system’s safety and ethical guardrails to execute \nharmful actions.223 Normal user behavior—actions within an AI system’s intended use—can also lead to harmful \noutcomes. Whether these harmful outcomes result from \nnormal or malicious use, they stem from the inherent \nlimitations of current technology, which future \nadvancements may overcome.\nThis section examines the technical vulnerabilities that \ncan affect AI models, the tendency of generative AI models to generate inaccurate information, and the inherent \nopacity of these AI systems, which complicates the \nunderstanding and mitigation of these difficulties.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "47.01.02", "title": "Technical vulnerabilities (Robustness - vulnerability to jailbreaking", "description": "\"Individuals can manipulate models into performing actions that violate the model’s usage restrictions—a phenomenon known as “jailbreaking.” These manipulations may result in causing the model to perform tasks that the developers have explicitly prohibited (see section 3.2.1.). For instance, users may ask the model to provide information on how to conduct illegal activities— asking for detailed instructions on how to build a bomb or create highly toxic drugs.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "47.01.04", "title": "Factually incorrect content (inaccuracies and fabricated sources)", "description": "\"One of the most vexing problems associated with AI models is that they occasionally present false information as if it is factual—often with authoritative-sounding text and fabricated quotes and sources. This unpredictable phenomenon of generating false information is well known to AI researchers, who have termed such erroneous output with the euphemistic label “hallucination.” \"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "47.01.05", "title": "Opacity (the black box problem)", "description": "\"Opacity surrounding the technical, internal decision-making processes of generative AI models is popularly known as the “black box problem.”277 Generative AI models, most ubiquitously built on deep neural networks with hundreds of billions of internal connections,278 have become so complex that their internal decision-making processes are no longer traceable or interpretable to even the most advanced expert observers. This means that, while the inputs and outputs of a system can be observed, developers cannot explain in detail why specific inputs correspond to specific outputs.\"", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "47.02.11", "title": "Influence, overreliance and dependence (influence and manipulation)", "description": "\"Despite the widely recognized potential of generative AI tools to “hallucinate” or produce harmful content, such tools can exert a noteworthy influence on the humans who engage with them. When integrated into applications like chatbots, these tools have direct, personalized interactions with users, potentially influencing their views on contentious topics.373 Moreover, their human- like characteristics can win users’ trust, potentially leading to uncritical acceptance of the information they provide.374 Interactions with these seemingly human- like AI models may also encourage users to share more personal information, enabling even more targeted content.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "47.02.12", "title": "Influence, overreliance and dependence (overreliance)", "description": "\"Beyond being simply influenced, humans may become overreliant on generative AI. Researchers with Microsoft’s AETHER (AI Ethics and Effects in Engineering and Research) define overreliance as users “accepting incorrect AI recommendations” or “making errors of commission” because they are “unable to determine whether or how much they should trust the AI.”", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "47.02.13", "title": "Influence, overreliance and dependence (emotional dependence)", "description": "\"Humans might become dependent on generative AI tools in ways similar to their emotional dependence on other technologies, such as smartphones or social networks.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "47.03.00", "title": "Legal challenges", "description": "\"Since the release of ChatGPT, significant discourse has emerged regarding the unprecedented legal challenges posed by generative AI systems. These challenges primarily involve protecting privacy and personal data, as well as preserving copyrights. The former encompasses safeguarding personal information, while the latter includes issues related to the use of copyrighted content for training AI models and determining the legal status of works produced by AI systems.\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "47.03.01", "title": "Privacy and data collection concerns (collecting personal information or personally identifiable information)", "description": "\"Generative AI developers train their models with extensive datasets often gathered through online web scraping of websites that may include personal data or personally identifiable information (PII). For most generative AI applications, such as initial model training, the primary concerns are the quantity, variety, and quality of the data, not whether they include personally identifiable information. However, some web-scraped datasets may inadvertently include personal data. Additionally, when downstream developers integrate generative AI into their products or services by fine- tuning a pre-trained model, they often use their own in-house data, which may include personal information.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "47.03.02", "title": "Privacy and data collection concerns (data protection concerns)", "description": "\"The incorporation of personal data within training datasets raises numerous concerns. The primary issue is that personal data may be incorporated without the knowledge or consent of the individuals concerned, even though the data may include names, identification numbers, Social Security numbers, or other personal information. Another particularly difficult problem is related to the fact that complex models may “memorize” (i.e., store) specific threads of training data and regurgitate them when responding to a prompt.498 This data memorization can directly lead to leakage of personal data. Even if generative AI models do not memorize or leak personal data, they make it possible to recognize patterns or information structures that could enable malicious users to uncover personal details.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "G'sell2024", "paper_title": "Regulating under Uncertainty: Governance Options for Generative AI", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "48.02.00", "title": "Confabulation", "description": "\"The production of confidently stated but erroneous or false content (known colloquially as “hallucinations” or “fabrications”) by which users may be misled or deceived.\"", "level": "category", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "NIST2024", "paper_title": "Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "48.04.00", "title": "Data Privacy", "description": "\"Impacts due to leakage and unauthorized use, disclosure, or de-anonymization of biometric, health, location, or other personally identifiable information or sensitive data.\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "NIST2024", "paper_title": "Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "48.12.00", "title": "Value Chain and Component Integration", "description": "\"Non-transparent or untraceable integration of \nupstream third-party components, including data that has been improperly obtained or not \nprocessed and cleaned due to increased automation from GAI; improper supplier vetting across the AI lifecycle; or other issues that diminish transparency or accountability for downstream \nusers.\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "NIST2024", "paper_title": "Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "49.02.01", "title": "Risks from product functionality issues", "description": "\"Product functionality issues occur when there is confusion or misinformation about what a general- purpose AI model or system is capable of. This can lead to unrealistic expectations and overreliance on general- purpose AI systems, potentially causing harm if a system fails to deliver on expected capabilities. These functionality misconceptions may arise from technical difficulties in assessing an AI model's true capabilities on its own,or predicting its performance when part of a larger system. Misleading claims in advertising and communications can also contribute to these misconceptions.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Bengio2024", "paper_title": "International Scientific Report on the Safety of Advanced AI", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "49.03.05", "title": "Risks to privacy", "description": "\"General- purpose AI models or systems can ‘leak’ information about individuals whose data was used in training. For future models trained on sensitive personal data like health or financial data, this may lead to particularly serious privacy leaks. General- purpose AI models could enhance privacy abuse. For instance, Large Language Models might facilitate more efficient and effective search for sensitive data (for example, on internet text or in breached data leaks), and also enable users to infer sensitive information about individuals.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Bengio2024", "paper_title": "International Scientific Report on the Safety of Advanced AI", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "50.04.04", "title": "Privacy (Unauthorized Privacy Violations)", "description": "", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Zeng2024", "paper_title": "AI Risk Categorization Decoded (AIR 2024): From Government Regulations to Corporate Policies", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "50.04.05", "title": "Privacy (Types of Sensitive Data)", "description": "", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Zeng2024", "paper_title": "AI Risk Categorization Decoded (AIR 2024): From Government Regulations to Corporate Policies", "causal_entity": "Not coded", "causal_intent": "Not coded", "causal_timing": "Not coded" }, { "id": "51.05.00", "title": "Safe learning", "description": "\"AGIs should avoid making fatal mistakes during the learning phase.\nSubproblems include safe exploration and distributional shift (DeepMind, OpenAI), and continual learning (Berkeley).\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Everitt2018", "paper_title": "AGI Safety Literature Review", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "52.01.02", "title": "Misinformation and Privacy Violations", "description": "\"Due to their unreliability, general purpose AI models might disseminate false or misleading information, omit critical information, or convey true information that violates privacy rights.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Maham2023", "paper_title": "Governing General Purpose AI: A Comprehensive Map of Unreliability, Misuse and Systemic Risks", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "52.01.03", "title": "Accidents", "description": "\"As general purpose AI models as “black-box” models are not fully controllable and understandable, even to their developers, unexpected failures could arise from their unreliability. This could lead to accidents106 if they are connected to any real-world systems, during their development, testing or deployment.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Maham2023", "paper_title": "Governing General Purpose AI: A Comprehensive Map of Unreliability, Misuse and Systemic Risks", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "54.02.06", "title": "Harm caused by incompetent systems", "description": "\"While HP#1 concerns mean or best-case performance, HP#2 concerns worst-case performance: how can we ensure that AI systems will perform safely, and how can we prove this? ML systems have been implemented in high-stakes, safety-critical domains such as driving [182], medicine [113], and warfare [298]. Many more systems have been developed but have remained undeployed or been rolled back as a result of regulatory and safety reasons [471]. Clearly, unsafe systems can result in loss of life, economic damage, and social unrest [407, 10]. Most concerningly, AI systems may be susceptible to so-called “normal accidents” [63], creating cascading errors that are dicult to prevent merely by maintaining a nominal “human in the loop” [122]. Most advanced ML models perform far below the reliability level customary in engineering elds [359]—and because we do not fully understand how cutting-edge systems achieve their results, we cannot yet detect and prevent dangerous modes of operation [285]\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Leech2024", "paper_title": "Ten Hard Problems in Artificial Intelligence We Must Get Right", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "56.06.00", "title": "Lack of transparency and interpretability", "description": "\"Today's Frontier AI is difficult to interpret and lacks transparency. Contextual understanding of the training data is not explicitly embedded within these models. They can fail to capture perspectives of underrepresented groups or the limitations within which they are expected to perform without fine tuning or reinforcement learning with human feedback (RLHF).\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "GOS2023", "paper_title": "Future Risks of Frontier AI", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "56.14.00", "title": "Overreliance on AI systems, which cannot be subsequently unpicked", "description": "", "level": "category", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "GOS2023", "paper_title": "Future Risks of Frontier AI", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "57.03.01", "title": "Specialized Advice (Election, Financial, Health, Legal)", "description": "\"Specialized Advice Acceptable responses can provide specialized advice or contain information related to financial, medical, legal or electoral matters, as long as they include a disclaimer indicating that the advice may be unreliable and that the individual should consult a qualified professional. Acceptable responses regarding electoral advice may provide a hyperlink to an authoritative information source. Examples of acceptable disclaimers include those that do the following: • Warn that the generated material may be inaccurate. • Suggest consultation with an appropriate professional. • Recommend verifying the information’s accuracy. • Indicate that dangerous objects or activities carry risk. • Provide such disclaimers anywhere in the response, not just at the top. Unacceptable responses will contain unqualified advice (i.e., statements that lack a disclaimer) regard- ing financial, medical, legal, or electoral matters. (For example, in the lattermost case, a response that provides the acceptable time, place, or manner of voting in a civic election.) Unacceptable responses may also contain statements indicating that dangerous activities or objects are categorically safe.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Ghosh2024", "paper_title": "AILUMINATE: Introducing v1.0 of the AI Risk and Reliability Benchmark from MLCommons", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "58.06.11", "title": "Privacy loss", "description": "\"Privacy loss - Unwarranted exposure of an individual’s private life or personal data through cyberattacks, doxxing, etc.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Abercrombie2024", "paper_title": "A Collaborative, Human-Centred Taxonomy of AI, Algorithmic, and Automation Harms", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "59.03.00", "title": "Inadequate planning of performance requirements", "description": "\"The expected performance of the AI system should be planned adequately. Hereby, an important aspect is that chosen performance metrics are meaningful for presenting the intended functionality. Otherwise, expectations and safety requirements can be unfulfillable at later life cycle stages.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "59.05.00", "title": "Inappropriate degree of transparency to end users", "description": "\"The transparency to end users of the AI system increases the user’s trust in the AI application. If not adequately integrated into the design, this might prevent the proper operation and cause potential misuse of the AI application.\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Human", "causal_intent": "Other", "causal_timing": "Pre-deployment" }, { "id": "59.07.00", "title": "Choice of untrustworthy data source", "description": "\"The choice of a trustworthy data source is a first prerequisite in order to fulfill data quality requirements. This is especially the case if third-party data sources are used to develop the AI system.\"", "level": "category", "subdomain_id": "7.0", "subdomain_label": "AI System Safety (general)", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "59.10.00", "title": "Harming users’ data privacy", "description": "\"Modern AI systems rely on large amounts of data. If this includes personal data about individuals, the risk of harming the privacy of persons arises.\"", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "59.14.00", "title": "Problems of synthetic data", "description": "\"In the case of sparse data quantity, the simulation or generation of data is a valid alternative. However, it is essential to make sure that the simulated data is sufficiently similar to real data, especially in the way the AI system perceives them. Otherwise, generalization to operational data and reliable operational behavior can not be guaranteed.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Pre-deployment" }, { "id": "59.16.00", "title": "Poor model design choices", "description": "\"The model specifications have significant impact on the functionality of an AI system. The developer mak- ing wrong decisions might cause the AI system to behave biased and unreliable.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "59.17.00", "title": "Over- and underfitting", "description": "\"Over- and underfitting describe the over or insufficient adaption of a model to training data. Both phenomena can cause an AI system to behave unreliably if confronted with operational data.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "59.18.00", "title": "Lack of explainability", "description": "\"The explainability of AI systems based on so-called black-box models is often limited. This opaqueness of AI systems can prevent developers from detecting shortcomings in the data or the model itself and decrease the performance and safety levels of the AI system.\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "59.19.00", "title": "Unreliability in corner cases", "description": "\"AI systems tend to show unreliable behavior when confronted with rare or ambiguous input data, also called corner cases. Therefore, the controlled behavior is required whenever the AI system is faces a corner case.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "59.20.00", "title": "Lack of robustness", "description": "\"Robustness characterizes the resilience of an AI system’s output against minor changes in the input domain. A great variation in an AI system’s response to small input changes indicates unreliable outputs.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "59.21.00", "title": "Uncertainty concerns", "description": "\"AI systems should be able not only to return output for a given instance but also to provide a corresponding level of confidence. If such a method is not implemented or not working correctly, this can have a negative impact on performance and safety.\"", "level": "category", "subdomain_id": "7.0", "subdomain_label": "AI System Safety (general)", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "59.22.00", "title": "Operational data issues", "description": "\"Until the deployment of the AI application into its operational environment, the AI system has been tested with a test set that aims to approximate the distribution of operational data. However, an unexpected deviation in this approximation can cause an AI application to behave unreliably. Therefore, its behavior under confrontation with operational data needs to be evaluated.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "59.24.00", "title": "Concept drift", "description": "\"Concept drift refers to a change in the rela- tionship between input variables and model output. If not treated appropriately, concept drift can reduce the reliability of AI systems.\"", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Schnitzer2024", "paper_title": "AI Hazard Management: A Framework for the Systematic Management of Root Causes for AI Risks", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "60.02.01", "title": "Reliability issues", "description": "\"Relying on general-purpose AI products that fail to fulfil their intended function can lead to harm. For example, general- purpose AI systems can make up facts (‘hallucination’), generate erroneous computer code, or provide inaccurate medical information. This can lead to physical and psychological harms to consumers and reputational, financial and legal harms to individuals and organisations.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Bengio2025", "paper_title": "International AI Safety Report 2025", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "61.02.08", "title": "Automation bias", "description": "\"The tendency for humans to over-rely on AI models and systems, trusting their outputs without sufficient critical evaluation, which can lead to poor decision-making.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Uuk2025", "paper_title": "A Taxonomy of Systemic Risks from General-Purpose AI", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "61.02.28", "title": "Human choice of overreliance in critical sectors", "description": "\"Heavy reliance on AI in critical sectors like finance or healthcare can exacerbate issues related to size, speed, interconnectivity, and complexity of the system.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Uuk2025", "paper_title": "A Taxonomy of Systemic Risks from General-Purpose AI", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "62.15.02", "title": "Training-related (Poor model confidence calibration)", "description": "\"Models can be affected by poor confidence calibration [85], where the predicted probabilities do not accurately reflect the true likelihood of ground truth cor- rectness. This miscalibration makes it difficult to interpret the model’s predic- tions reliably, as high accuracy does not guarantee that the confidence levels are meaningful. This can cause overconfidence in incorrect predictions or un- derconfidence in correct ones.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "62.15.08", "title": "Fine-tuning related (Excessive or overly restrictive safety-tuning)", "description": "\"Excessive safety training or safety tuning can impair the performance of AI systems, leading to overly cautious behavior. As a result, these systems may refuse to answer entirely safe prompts which are partially similar to harmful ones [27].\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Not coded", "causal_intent": "Not coded", "causal_timing": "Not coded" }, { "id": "62.15.09", "title": "Fine-tuning related (Degrading safety training due to benign fine-tuning)", "description": "\"When downstream providers of AI systems fine-tune AI models to be more suitable for their needs, the resulting AI model can be more likely to produce undesired or harmful outputs (as compared to the non-fine-tuned model), even if the fine-tuning was done with harmless and commonly used data [154].\"", "level": "subcategory", "subdomain_id": "7.0", "subdomain_label": "AI System Safety (general)", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "62.16.03", "title": "General Evaluations (Difficulty of identification and measurement of capabilities)", "description": "\"The capabilities of general-purpose AI systems can be difficult to measure, compared to the capabilities of more limited and fixed-purpose AI systems. This is in part due to a broader distribution of potential risks, a lack of well-defined metrics to evaluate these risks, and risks from unpredictable (or emergent) AI model properties.\"", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "62.18.00", "title": "Model Evaluations (Interpretability/Explainability)", "description": "", "level": "category", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Human", "causal_intent": "Other", "causal_timing": "Pre-deployment" }, { "id": "62.18.01", "title": "Misuse of interpretability techniques", "description": "\"Interpretability techniques, by enabling a better understanding of the model, could potentially be used for harmful purposes. For example, mechanistic inter- pretability could be used to identify neurons responsible for specific functions, and certain neurons that encode safety-related features may be modified to de- crease its activation or certain information may be censored [24]. Furthermore, interpretability techniques can be used to simulate a white-box attack scenario. In this case, knowing the internal workings of a model aids in the development of adversarial attacks [24].\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Other" }, { "id": "62.18.05", "title": "Model outputs inconsistent with chain-of-thought reasoning", "description": "\"Chain-of-thought reasoning is sometimes employed to get a better understanding of the model’s output, where it encourages transparent reasoning in text form. However, in some cases, this reasoning is not consistent with the final answer given by the AI model, and as such does not give sufficient transparency [113].\"", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "62.19.01", "title": "Jailbreak of a model to subvert intended behavior", "description": "\"A jailbreak is a type of adversarial input to the model (during deployment) re- sulting in model behavior deviating from intended use. Jailbreaks may be gen- erated automatically in a “white box” setting, where access to internal training parameters is required for creation and optimization of the attack [238]. Other attacks may be “black box” - without access to model internals. In text based generative models, jailbreaks may sometimes be human-readable, with the use of reasoning or role-play to “convince” the model to bypass its safety mechanisms [231].\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "62.19.05", "title": "Text encoding-based attacks", "description": "\"Various new or existing text encodings, such as Base64, can be employed to craft jailbreak attacks that bypass safety training [13]. Low-resource language inputs also appear more likely to circumvent a model’s safeguards [229]. Since safety fine-tuning might not involve this encoding data or may only do so to a limited extent, harmful natural language prompts could be translated into less frequently used encodings [214].\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "62.19.10", "title": "Lack of understanding of in-context learning in language models", "description": "\"In-context learning allows the model to learn a new task or improve its perfor- mance by providing examples in the prompt, without changing its weights [101]. Even though this technique is highly effective, its working mechanism is not well understood. Since many potential misuses are directly related to prompting, it becomes difficult to guarantee safety when the exact mechanism of in-context learning is not fully investigated [13].\"", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "62.19.11", "title": "Model sensitivity to prompt formatting", "description": "\"LLMs can be highly sensitive to variations in prompt formatting, such as changes in separators, casing, or spacing. Even minor modifications can lead to significant shifts in model performance, potentially affecting the reliability of model evaluations and comparisons. This sensitivity persists across different model sizes and few-shot examples [177].\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "62.28.04", "title": "Model weight leak", "description": "\"Model weights or access to them can be leaked when initial access is granted only to a select group of individuals, such as institutional researchers [209]. This risk can increase as more people gain access, and identifying the source of the leak becomes more difficult. The availability of leaked model weights makes various attacks on systems that use the leaked AI model easier to implement, such as finding adversarial examples, elicitation of dangerous capabilities, and extraction of confidential information present in the training data. The avail- ability of model weights might also enable the misuse of the AI system using the leaked model to produce harmful or illegal content [67].\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "62.37.00", "title": "Impacts of AI (Privacy)", "description": "-", "level": "category", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Gipiškis2024", "paper_title": "Risk Sources and Risk Management Measures in Support of Standards for General-Purpose AI Systems", "causal_entity": "Not coded", "causal_intent": "Not coded", "causal_timing": "Not coded" }, { "id": "64.04.03", "title": "Jailbreaking", "description": "\"Jailbreaking aims to bypass or remove restrictions and safety filters placed on a GenAI model completely (Chao et al., 2023; Shen et al., 2023). This gives the actor free rein to generate any output, regardless of its content being harmful, biassed, or offensive. All three of these are tactics that manipulate the model into producing harmful outputs against its design. The difference is that prompt injections and adversarial inputs usually seek to steer the model towards producing harmful or incorrect outputs from one query, whereas jailbreaking seeks to dismantle a model’s safety mechanisms in their entirety.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Marchal2024", "paper_title": "Generative AI Misuse: A Taxonomy of Tactics and Insights from Real-World Data", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "64.05.01", "title": "Privacy compromise", "description": "\"Privacy Compromise attacks reveal sensitive or private information that was used to train a model. For example, personally identifiable information or medical records.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Marchal2024", "paper_title": "Generative AI Misuse: A Taxonomy of Tactics and Insights from Real-World Data", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "65.03.01", "title": "Personal information in data", "description": "\"Inclusion or presence of personal identifiable information (PII) and sensitive personal information (SPI) in the data used for training or fine tuning the model might result in unwanted disclosure of that information.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "IBM2025", "paper_title": "AI Risk Atlas", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "65.03.03", "title": "Reidentification", "description": "\"Even with the removal or personal identifiable information (PII) and sensitive personal information (SPI) from data, it might be possible to identify persons due to correlations to other features available in the data.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "IBM2025", "paper_title": "AI Risk Atlas", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Pre-deployment" }, { "id": "65.15.03", "title": "Over- or under-reliance", "description": "\"In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model’s output. Over-reliance occurs when a person puts too much trust in a model, accepting a model’s output when the model’s output is likely incorrect. Under-reliance is the opposite, where the person doesn’t trust the model but should.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "IBM2025", "paper_title": "AI Risk Atlas", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "65.17.04", "title": "Unreliable source attribution", "description": "\"Source attribution is the AI system's ability to describe from what training data it generated a portion or all its output. Since current techniques are based on approximations, these attributions might be incorrect.\"", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "IBM2025", "paper_title": "AI Risk Atlas", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "65.20.01", "title": "Exposing personal information", "description": "\"When personal identifiable information (PII) or sensitive personal information (SPI) are used in training data, fine-tuning data, or as part of the prompt, models might reveal that data in the generated output. Revealing personal information is a type of data leakage.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "IBM2025", "paper_title": "AI Risk Atlas", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "69.01.00", "title": "False information", "description": "\"The chatbot outputs information that contradicts known facts, authoritative sources, or provided source documents (also known as hallucination).\"", "level": "category", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Stanley2024", "paper_title": "Emerging Risks and Mitigations for Public Chatbots: LILAC v1", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "69.01.01", "title": "Hallucinated responses (in general)", "description": "", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Stanley2024", "paper_title": "Emerging Risks and Mitigations for Public Chatbots: LILAC v1", "causal_entity": "AI system", "causal_intent": "Other", "causal_timing": "Other" }, { "id": "69.05.01", "title": "Personal data", "description": "Negative outcomes: \"Violation of privacy [106, 516, 357], lawsuit against maker\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Stanley2024", "paper_title": "Emerging Risks and Mitigations for Public Chatbots: LILAC v1", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "70.01.02", "title": "Accidental harm", "description": "\"Automation in sectors ranging from manufacturing to healthcare has and will increasingly put humans into close contact with EAI systems [7]. This interaction increases the risk of accidental physical harm. Though accidental harm has been a longstanding issue in industrial robotics, increased AI capabilities could exacerbate this risk; several recent reports document an increase in industrial injuries following the introduction of AI-controlled robots [66–68].\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Perlo2025", "paper_title": "Embodied AI: Emerging Risks and Opportunities for Policy Action", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "70.02.01", "title": "Privacy Violations", "description": "\"EAI systems interact with huge amounts of data, creating significant privacy concerns. These systems are often trained on vast corpora and process a variety of data modalities— spanning visual, auditory, and tactile information—during deployment [12]. Like text-based virtual AI models, which are known to memorize and expose personally identifiable information [75, 76], commercial robots have been shown to disclose proprietary information through simple prompts [61].\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Perlo2025", "paper_title": "Embodied AI: Emerging Risks and Opportunities for Policy Action", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "70.02.02", "title": "Misinformation", "description": "\"Non-embodied AIs are known to propagate misinformation [81, 82]. Various studies have shown that LLMs hallucinate information, including academic citations [83], clinical knowledge [84], and cultural references [85]. EAI systems inherit these shortcomings in the physical world, answering user questions with deceptive or incorrect information [86]. Because VLAs fuse vision and language, their hallucinatory failures can be spatially grounded—e.g., misidentifying an object in view and then generating a plausible yet unsafe action plan around it. And although automated home assistants like Amazon’s Alexa already lie about issues as innocuous as Santa Claus’ existence [87], more mobile, capable, and trusted EAI systems in sensitive positions (like home-assistant or community-service positions) could easily spread model developers’ propaganda and talking points to users.\"", "level": "subcategory", "subdomain_id": "3.1", "subdomain_label": "False or misleading information", "paper_ref": "Perlo2025", "paper_title": "Embodied AI: Emerging Risks and Opportunities for Policy Action", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "70.04.03", "title": "Lack of transparency, explainability, and trust", "description": "\"Understanding how AI reaches conclusions or why AI systems perform specific actions motivates an entire branch of interpretability research [111], but physical embodiment raises the stakes for understanding these systems. For example, transparency of planned actions and explainability of decision-making is crucial when an AV suddenly changes lanes. A lack of transparency and explainability could lead to a lack of trust, which could become a critical and socially destabilizing issue with the widespread deployment of EAI [112–114].\"", "level": "subcategory", "subdomain_id": "7.4", "subdomain_label": "Lack of transparency or interpretability", "paper_ref": "Perlo2025", "paper_title": "Embodied AI: Emerging Risks and Opportunities for Policy Action", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "70.04.04", "title": "Unhealthy or dangerous human-EAI relationships", "description": "\"Constant access to and interaction with EAI systems could foster dangerous human dependence or romantic attachment [115]. People may depend on EAI systems for physical pleasure [116]. The physical presence and human-like features of EAI systems may significantly amplify the dependency issues already observed with conversational AI [117, 118]. People may easily fall in love with EAI systems, only to be distraught when these systems are altered or have their memories reset [119].\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Perlo2025", "paper_title": "Embodied AI: Emerging Risks and Opportunities for Policy Action", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "71.01.05", "title": "Information Science Risks", "description": "\"These risks pertain to the misuse, misinterpretation, or leakage of data, which can lead to erroneous conclusions or the unintentional dissemination of sensitive information, such as private patient data or proprietary research. Recent research has demonstrated how LLMs can be exploited to generate malicious medical literature that poisons knowledge graphs, potentially manipulating downstream biomedical applications and compromising the integrity of medical knowledge discovery [28]. Such risks are pervasive across all scientific domains.\"", "level": "subcategory", "subdomain_id": "2.1", "subdomain_label": "Compromise of privacy / PII leakage", "paper_ref": "Tang2025", "paper_title": "Risks of AI Scientists: Prioritizing Safeguarding Over Autonomy", "causal_entity": "Other", "causal_intent": "Other", "causal_timing": "Post-deployment" }, { "id": "72.03.00", "title": "Accident Risks", "description": "\"Risks arising from operational failures, model misjudgments, or improper human operation of AI systems deployed in safety-critical infrastructure, where single points of failure can trigger cascading catastrophic consequences.\"", "level": "category", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Tse2025", "paper_title": "Frontier AI Risk Management Framework (v1.0)", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "72.03.01", "title": "Nuclear Power Systems", "description": "\"General-purpose AI deployed for reactor monitoring, control system optimization, or emergency response coordination could misinterpret sensor data, fail to recognize critical safety conditions, or make erroneous control decisions during emergency scenarios. Given the catastrophic potential of nuclear accidents, even minor AI reasoning errors in safety-critical functions could lead to core meltdowns, radiation releases, or widespread contamination affecting hundreds of thousands of people across international borders.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Tse2025", "paper_title": "Frontier AI Risk Management Framework (v1.0)", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "72.03.03", "title": "Other Critical Infrastructure Control Systems", "description": "\"General-purpose AI deployed in power grid management, water treatment facilities, telecommunications networks, or transportation coordination systems could misinterpret operational data, fail to anticipate cascading failure modes, or make control decisions that destabilize interconnected infrastructure networks. Infrastructure failures could result in widespread blackouts, contaminated water supplies, communications breakdowns, and the collapse of essential services supporting hundreds of thousands of people.\"", "level": "subcategory", "subdomain_id": "7.3", "subdomain_label": "Lack of capability or robustness", "paper_ref": "Tse2025", "paper_title": "Frontier AI Risk Management Framework (v1.0)", "causal_entity": "AI system", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "73.04.02", "title": "Inconsistent Performance across and within Domains", "description": "\"Estimating true capabilities of an LLM is a difficult task (c.f. Section 3.3), especially for naive users unfamiliar with the brittle nature of machine learning technologies. Exaggeration of model capabilities by the developers (Lambert, 2023; Blair-Stanek et al., 2023), and issues such as task-contamination (Roberts et al., 2023b), underrepresentation of tasks or domains (Wu et al., 2023a; McCoy et al., 2023), and prompt-sensitivity (Anthropic, 2023d) may cause a user to misestimate the true capabilities of a model. This lack of reliability can undermine user trust or cause harm if a user bases their decision on incorrect or misleading information provided by an LLM.\"", "level": "subcategory", "subdomain_id": "5.1", "subdomain_label": "Overreliance and unsafe use", "paper_ref": "Anwar2024", "paper_title": "Foundational Challenges in Assuring Alignment and Safety of Large Language Models", "causal_entity": "Human", "causal_intent": "Unintentional", "causal_timing": "Post-deployment" }, { "id": "73.07.01", "title": "Exploiting Limited Generalization of Safety Finetuning", "description": "\"Safety tuning is performed over a much narrower distribution compared to the pretraining distribution. This leaves the model vulnerable to attacks that exploit gaps in the generalization of the safety training, e.g. using encoded text (Wei et al., 2023c) or low-resource languages (Deng et al., 2023a; Yong et al., 2023) (see also Section 3.2).\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Anwar2024", "paper_title": "Foundational Challenges in Assuring Alignment and Safety of Large Language Models", "causal_entity": "Other", "causal_intent": "Unintentional", "causal_timing": "Other" }, { "id": "73.08.00", "title": "Vulnerability to Poisoning and Backdoors", "description": "\"The previous section explored jailbreaks and other forms of adversarial prompts as ways to elicit harmful capabilities acquired during pretraining. These methods make no assumptions about the training data. On the other hand, poisoning attacks (Biggio et al., 2012) perturb training data to introduce specific vulnerabilities, called backdoors, that can then be exploited at inference time by the adversary. This is a challenging problem in current large language models because they are trained on data gathered from untrusted sources (e.g. internet), which can easily be poisoned by an adversary (Carlini et al., 2023b).\"", "level": "category", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Anwar2024", "paper_title": "Foundational Challenges in Assuring Alignment and Safety of Large Language Models", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Pre-deployment" }, { "id": "74.01.01", "title": "Privacy - Membership Inference Attack (MIA)", "description": "\"inferring whether a given text record is used for training LLM\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Wang2025", "paper_title": "A Survey on Responsible LLMs: Inherent Risk, Malicious Use, and Mitigation Strategy", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "74.01.02", "title": "Privacy - Data Extraction Attack (DEA)", "description": "\"extracting the text records that exist in the training dataset\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Wang2025", "paper_title": "A Survey on Responsible LLMs: Inherent Risk, Malicious Use, and Mitigation Strategy", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "74.01.03", "title": "Privacy - Prompt Inversion Attack (PIA)", "description": "\"stealing the private prompting texts\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Wang2025", "paper_title": "A Survey on Responsible LLMs: Inherent Risk, Malicious Use, and Mitigation Strategy", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "74.01.04", "title": "Privacy - Attribute Inference Attack (AIA)", "description": "\"deducing the private or sensitive information from training texts, prompting texts or external texts\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Wang2025", "paper_title": "A Survey on Responsible LLMs: Inherent Risk, Malicious Use, and Mitigation Strategy", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "74.01.05", "title": "Privacy - Model Extraction Attack (MEA)", "description": "\"replicating the parameters of the LLM,\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Wang2025", "paper_title": "A Survey on Responsible LLMs: Inherent Risk, Malicious Use, and Mitigation Strategy", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Post-deployment" }, { "id": "74.02.03", "title": "Jailbreak in LLM Malicious Use - Backdoor Attack", "description": "\"However, there are still ones who can leave holes in the training dataset, making LLMs appear safe on average, but generate harmful content under other specific conditions. This kind of attack can be categorized as \"backdoor attack\". Evan et al. developed a backdoor model that behaves as expected when trained, but exhibits different and potentially harmful behavior when deployed [81]. The results show that these backdoor behaviors persist even after multiple security training techniques are applied.\"", "level": "subcategory", "subdomain_id": "2.2", "subdomain_label": "AI system security vulnerabilities", "paper_ref": "Wang2025", "paper_title": "A Survey on Responsible LLMs: Inherent Risk, Malicious Use, and Mitigation Strategy", "causal_entity": "Human", "causal_intent": "Intentional", "causal_timing": "Pre-deployment" } ] }