@article{Abouelenein2025,
  title = {The R5E pattern: can artificial intelligence enhance programming skills development?},
  abstract = {The rapid proliferation of artificial intelligence (AI), particularly within education, presents both opportunities and challenges. While AI offers innovative solutions to pedagogical challenges, unstructured utilization of AI tools like ChatGPT can foster passive learning, with students relying on automated solutions rather than engaging in active learning. This quasi-experimental study addressed this concern by introducing and evaluating the R5E pattern, a novel pedagogical model for integrating ChatGPT into programming education to promote active learning and critical thinking. The study compared the efficacy of the R5E pattern with unstructured ChatGPT use in developing programming skills among undergraduate students. Seventy students were randomly assigned to either an R5E group or an unstructured ChatGPT using group. A mixed-methods approach, employing the Programming Skills Cognitive Test (PS-CT) and the Programming Skills Performance Observation Card (PS-POC), assessed both cognitive and performance aspects of programming skills. Pre- and post-intervention data revealed improved programming skills across both groups; however, statistically significant differences favoured the R5E group on both post-intervention assessments. A significant positive correlation between PS-CT and PS-POC scores indicated that increased theoretical knowledge corresponded with enhanced practical performance. Qualitative observations revealed that R5E students engaged in more peer and instructor interaction, alongside ChatGPT use, and posed more sophisticated questions, unlike the second group, which relied more heavily on direct ChatGPT queries. These findings support the superior efficacy of the structured R5E pattern in fostering programming skill development compared to unstructured ChatGPT use. While limitations, including sample size and a short post-test timeframe, necessitate further research, this study suggests the R5E pattern holds promise as a validated pedagogical model for integrating AI into education. Future research should expand the sample, extend the intervention duration, and utilize a delayed post-test.},
  volume = {30},
  ISSN = {1573-7608},
  url = {http://dx.doi.org/10.1007/s10639-025-13616-3},
  DOI = {10.1007/s10639-025-13616-3},
  number = {15},
  journal = {Education and Information Technologies},
  publisher = {Springer Science and Business Media LLC},
  author = {Abouelenein, Yousri Attia Mohamed and Ghazala, Ayat Fawzy Ahmed and Mahdy, Eman Mahdy Mohamed and Khalaf, Mohamed Hassan Ragab},
  year = {2025},
  month = June,
  pages = {22177–22205},
  critique = {This is a quasi-experiment with no true control group: both of its two groups used ChatGPT, so it can only show that a structured (R5E) prompt pattern beats unstructured ChatGPT use, not that ChatGPT improves programming skill development as the title implies. The sample is small and homogeneous (70 students, all male, from a single Egyptian college), and the two instruments (PS-CT and PS-POC) were developed by the authors with limited reported psychometric validation. There is an instructor confound: the two groups' practical sessions were run by different instructors, each trained only in their own group's approach, so instructor effects are entangled with the treatment. The significant PS-CT/PS-POC correlation is a trivially expected finding, and with only immediate post-tests (no delayed retention) plus qualitative observation by the intervention's designers, the superiority claim is vulnerable to expectancy and demand effects.}
}

@inproceedings{Adeeb2025,
  title = {How Do Novice Programmers Solve Code-Tracing Problems When ChatGPT Is Available? A Qualitative Analysis},
  abstract = {Large Language Models (LLMs) such as ChatGPT are generative AI that can answer questions and provide solutions to problems in various domains. To date, little is known about how students interact with LLM tools while problem solving. We recruited novice programmers (N = 21) and gave them a brief video-based lesson on the fundamentals. We then asked them to use ChatGPT version GPT-4o to (1) learn about an additional construct, namely for loops, and (2) solve four problems that required specifying the output of a program with a for loop, with access to ChatGPT. Participants verbalized their thoughts as they worked, and these were transcribed and analyzed. We used a qualitative approach to identify student behaviors and strategies during code tracing. We found substantially more independent problem solving than expected given prior reports on students’ over-reliance on LLMs.},
  author = {Adeeb, Elmira and Muldner, Kasia},
  booktitle = {Proceedings of the 2025 ACM Conference on International Computing Education Research V.1},
  publisher = {ACM},
  year = {2025},
  month = Aug,
  pages = {421–434},
  DOI = {10.1145/3702652.3744207},
  url = {https://doi.org/10.1145/3702652.3744207},
  critique = {This is a small, single-condition qualitative study (N=21 novice programmers thinking aloud) with no control or comparison group, so it cannot establish whether ChatGPT helps or hinders learning, only describe how these particular students behaved. The authors themselves concede they could not analyze the association between strategies and learning because of the design and modest sample. The headline finding of 'more independent problem solving than expected' is a comparison against a prior, different study (code generation, not tracing) rather than a controlled contrast, and the pre/post gains have no baseline group to which learning can be attributed. Self-selected volunteers from one institution limit generalizability, and the study is confined to code tracing, which the authors note is simpler than code generation.}
}

@article{Akcapinar2024,
  title = {AI chatbots in programming education: guiding success or encouraging plagiarism},
  abstract = {This study examines the impact of an AI programming assistant on students' exam scores and their tendency to accept incorrect AI-generated information. The customized AI programming assistant was developed by the authors using GPT based Large Language Model (LLM). A one group pretest–posttest quasi-experimental design was utilized to answer research questions. Students were asked to take identical programming exams twice: once without AI assistance and once with the option to use the AI assistant. Results showed that the students’ average exam scores significantly increased from 48.33 to 74.47 with a large effect size (d = 1.56) when they used the AI assistance. On the other hand, when student—AI interaction logs were analyzed for a specific question, it was found that AI generated incorrect answers to 36 students. Thirty-three of these students (92\%) answered the question incorrectly. Even more interestingly, despite the AI-generated response containing an obvious error, 22 of them (61\%) copied and pasted the AI's response directly into the answer field. Only 3 students (8\%) ignored the incorrect response generated by the AI and answered the question correctly. A significant portion of students accepting incorrect information provided by AI underscores the need for careful integration of AI tools into learning environments. Moreover, our findings emphasize the importance of specially developed AI tools rather than free tools like ChatGPT in exploring the new type of interaction between students and AI.},
  volume = {4},
  ISSN = {2731-0809},
  url = {http://dx.doi.org/10.1007/s44163-024-00203-7},
  DOI = {10.1007/s44163-024-00203-7},
  number = {1},
  journal = {Discover Artificial Intelligence},
  publisher = {Springer Science and Business Media LLC},
  author = {Akçapınar, Gökhan and Sidan, Elif},
  year = {2024},
  month = Nov,
  critique = {The study uses a one-group pretest-posttest design in which the same 13-question exam is administered twice, so the large reported gain (48.33 to 74.47, d=1.56) is confounded with practice and familiarity effects: students see the identical items a second time and know their first attempt, and the midterm grade is the average of the two attempts, giving them an incentive to improve. There is no control group and no counterbalancing, so the improvement cannot be attributed to the AI assistant. The second finding, that 92\% of students accepted an incorrect AI answer, comes from a single deliberately planted trick question (a type conversion the GPT-3.5 backend gets wrong) rather than naturalistic use, and the 45-student, single-institution sample limits generalizability. The recommendation for specially developed AI tools over free tools is asserted from this one bounded comparison.}
}

@article{Alanazi2025a,
  title = {PyChatAI: Enhancing Python Programming Skills—An Empirical Study of a Smart Learning System},
  abstract = {This paper presents strategies for effectively integrating AI tools into programming education and provides recommendations for enhancing student learning outcomes through intelligent educational systems. Learning computer programming is a cognitively demanding task that requires dedication, logical reasoning, and persistence. Many beginners struggle with debugging and often lack effective problem-solving strategies. To address these issues, this study investigates PyChatAI—a bilingual, AI-powered chatbot designed to support novice Python programmers by providing real-time feedback, answering coding-related questions, and fostering independent problem-solving skills. PyChatAI offers continuous, personalised assistance and is particularly beneficial for students who prefer remote or low-pressure learning environments. An empirical evaluation employing a Solomon Four-Group design revealed significant improvements across all programming skill areas, with especially strong gains in theoretical understanding, code writing, and debugging proficiency.},
  volume = {14},
  ISSN = {2073-431X},
  url = {http://dx.doi.org/10.3390/computers14050158},
  DOI = {10.3390/computers14050158},
  number = {5},
  journal = {Computers},
  publisher = {MDPI AG},
  author = {Alanazi, Manal and Soh, Ben and Samra, Halima and Li, Alice},
  year = {2025},
  month = Apr,
  pages = {158},
  critique = {This companion paper is a near-duplicate of the authors' TAM3 study (same tool, same N=300 female students, same institution) repackaged as a 'smart learning system' evaluation, which raises salami-slicing concerns. It reports a Solomon Four-Group design, but the four groups in its sample-size table sum to only 200 (4 x 50) against the stated 300 participants, an unexplained inconsistency that undermines the reported allocation. The abstract's claim of 'significant improvements across all programming skill areas' is not tied to clearly specified, independently validated outcome measures, and the four-month, single-gender, single-institution design limits generalizability. Self-selection and the absence of any check that control groups avoided PyChatAI also threaten internal validity.}
}

@article{Alanazi2025b,
  title = {Examining the Influence of AI on Python Programming Education: An Empirical Study and Analysis of Student Acceptance Through TAM3},
  abstract = {This study investigates the adoption of PyChatAI, a bilingual AI-powered chatbot for Python programming education, among female computer science students at Jouf University. Guided by the Technology Acceptance Model 3 (TAM3), it examines the determinants of user acceptance and usage behaviour. A Solomon Four-Group experimental design (N = 300) was used to control pre-test effects and isolate the impact of the intervention. PyChatAI provides interactive problem-solving, code explanations, and topic-based tutorials in English and Arabic. Measurement and structural models were validated via Confirmatory Factor Analysis (CFA) and Structural Equation Modelling (SEM), achieving excellent fit (CFI = 0.980, RMSEA = 0.039). Results show that perceived usefulness (β = 0.446, p < 0.001) and perceived ease of use (β = 0.243, p = 0.005) significantly influence intention to use, which in turn predicts actual usage (β = 0.406, p < 0.001). Trust, facilitating conditions, and hedonic motivation emerged as strong antecedents of ease of use, while social influence and cognitive factors had limited impact. These findings demonstrate that AI-driven bilingual tools can effectively enhance programming engagement in gender-specific, culturally sensitive contexts, offering practical guidance for integrating intelligent tutoring systems into computer science curricula.},
  volume = {14},
  ISSN = {2073-431X},
  url = {http://dx.doi.org/10.3390/computers14100411},
  DOI = {10.3390/computers14100411},
  number = {10},
  journal = {Computers},
  publisher = {MDPI AG},
  author = {Alanazi, Manal and Li, Alice and Samra, Halima and Soh, Ben},
  year = {2025},
  month = Sept,
  pages = {411},
  critique = {This is a cross-sectional, self-report survey (N=300, all female, one Saudi university) analyzed with CFA/SEM, so the strong causal language ('significantly influence', 'predicts actual usage') over-reaches correlational TAM data. 'Actual usage' is self-reported rather than objectively measured, and all constructs come from the same questionnaire, inviting common-method variance that can inflate path coefficients despite good model fit. The study is essentially a TAM3 replication dressed up with an AI chatbot and offers little conceptual novelty, and its Solomon four-group structure does little for a one-shot acceptance survey where pretest sensitization is largely irrelevant. The findings are specific to a gender-segregated, bilingual, culturally specific context and establish no learning or engagement outcome, only stated intentions.}
}

@inproceedings{Azaiz2024,
  title = {Feedback-Generation for Programming Exercises With GPT-4},
  abstract = {Ever since Large Language Models (LLMs) and related applications have become broadly available, several studies investigated their potential for assisting educators and supporting students in higher education. LLMs such as Codex, GPT-3.5, and GPT 4 have shown promising results in the context of large programming courses, where students can benefit from feedback and hints if provided timely and at scale. This paper explores the quality of GPT-4 Turbo's generated output for prompts containing both the programming task specification and a student's submission as input. Two assignments from an introductory programming course were selected, and GPT-4 was asked to generate feedback for 55 randomly chosen, authentic student programming submissions. The output was qualitatively analyzed regarding correctness, personalization, fault localization, and other features identified in the material. Compared to prior work and analyses of GPT-3.5, GPT-4 Turbo shows notable improvements. For example, the output is more structured and consistent. GPT-4 Turbo can also accurately identify invalid casing in student programs' output. In some cases, the feedback also includes the output of the student program. At the same time, inconsistent feedback was noted such as stating that the submission is correct but an error needs to be fixed. The present work increases our understanding of LLMs' potential, limitations, and how to integrate them into e-assessment systems, pedagogical scenarios, and instructing students who are using applications based on GPT-4.},
  author = {Azaiz, Imen and Kiesler, Natalie and Strickroth, Sven},
  booktitle = {Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1},
  publisher = {ACM},
  year = {2024},
  month = July,
  pages = {31–37},
  DOI = {10.1145/3649217.3653594},
  url = {https://doi.org/10.1145/3649217.3653594}
}

@inproceedings{Benario2025,
  title = {Unlocking Potential with Generative AI Instruction: Investigating Mid-level Software Development Student Perceptions, Behavior, and Adoption},
  abstract = {Generative AI tools are rapidly evolving and impacting many domains, including programming. Computer Science (CS) instructors must address student access to these tools. While some advocate to ban the tools entirely, others suggest embracing them so that students develop the skills for utilizing the tools safely and responsibly. Studies indicate positive impacts, as well as cautions, on student outcomes when these tools are integrated into courses. We studied the impact of incorporating instruction on industry-standard generative AI tools into a mid-level software development course with students from 16 Minority Serving Institutions. 89\% of student participants used generative AI tools prior to the course without any formal instruction. After formal instruction, students most frequently used generative AI tools for explaining concepts and learning new things. Students generally reported positive viewpoints on their ability to learn to program and learn problem-solving skills while using generative AI tools. Finally, we found that students: reported to understand their code when they work with generative AI tools, are critical about the outputs that generative AI tools provide, and check outputs of generative AI tools to ensure accuracy.},
  author = {Benario, Jamie Gorson and Marroquin, Jenn and Chan, Monica M. and Holmes, Ernest D.V. and Mejia, Daniel},
  booktitle = {Proceedings of the 56th ACM Technical Symposium on Computer Science Education V. 1},
  publisher = {ACM},
  year = {2025},
  month = Feb,
  pages = {395–401},
  DOI = {10.1145/3641554.3701859},
  url = {https://doi.org/10.1145/3641554.3701859},
  critique = {This is a single-group pre/post design with no control group, so the authors themselves concede it 'cannot fully demonstrate cause and effect'; changes in self-reported behavior could reflect novelty, maturation, or the mere permission to use tools rather than the instruction. The sample is doubly self-selected: students had to apply and pass a technical assessment to join a Google-run program (atypical, motivated students, not representative of typical MSI students), and survey participation was voluntary with extra-credit incentives and ~24\% attrition (117 pre to 89 post, analyses restricted to n=88 who completed both). All outcomes are self-reported Likert/rating perceptions ('understand the code,' 'test the response'), with no objective measure of code quality or learning. The analysis runs ten pre/post t-tests on the behavior items without any multiple-comparison correction, inflating false-positive risk, and effect sizes are not reported; a single author coded the open-ended responses with no inter-rater reliability reported.}
}

@article{Haindl2024,
  title = {Does ChatGPT Help Novice Programmers Write Better Code? Results From Static Code Analysis},
  abstract = {In the realm of AI-enhanced programming education, there is growing interest in using such tools to help students understand good coding principles. This study investigates the impact of ChatGPT on code quality among part-time undergraduate students in introductory Java programming courses, who lack prior Java experience. The source code of 16 students from the control group (without ChatGPT) and 22 students from the treatment group (with ChatGPT) who completed identical programming exercises focused on coding conventions was analyzed. Static code analysis tools assessed adherence to a common coding convention ruleset and calculated cyclomatic and cognitive complexity metrics. The comparative analysis shows that the ChatGPT-assisted group significantly improved code quality, with fewer rule violations and reduced cyclomatic and cognitive complexities. The treatment group adhered more closely to coding standards and produced less complex code. Violations primarily occurred in line length, final parameters, and the extensibility of object-oriented programming (OOP). These findings suggest that ChatGPT can be beneficial in programming education by helping students write cleaner, less complex code and adhere to coding conventions. However, the study’s limitations, such as the small sample size and novice status of participants, call for further research with larger, more diverse populations and different educational contexts.},
  volume = {12},
  ISSN = {2169-3536},
  url = {http://dx.doi.org/10.1109/ACCESS.2024.3445432},
  DOI = {10.1109/access.2024.3445432},
  journal = {IEEE Access},
  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},
  author = {Haindl, Philipp and Weinberger, Gerald},
  year = {2024},
  pages = {114146–114156},
  critique = {The design is confounded rather than randomized: the control group (n=16) ran in summer 2022 (pre-ChatGPT) and the treatment group (n=22) in summer 2023, so any difference could reflect cohort, instructor, or content changes rather than ChatGPT. Treatment assignment was self-selected—students chose per exercise whether to use ChatGPT—and both groups are tiny and from a single institution, so the sample is neither randomized nor representative. Dozens of individual rules were tested with Wilcoxon rank-sum tests using an arbitrary p<=0.005 cutoff, but no multiple-comparison correction is reported across the many rules examined, inviting false positives. Code quality is reduced to static-analysis rule violations plus cyclomatic/cognitive complexity, which does not capture correctness or semantic quality. Repeated submissions per student across exercises appear treated as independent observations, ignoring within-student non-independence.}
}

@article{Jing2024,
  title = {What factors will affect the effectiveness of using ChatGPT to solve programming problems? A quasi-experimental study},
  abstract = {The emergence of ChatGPT has sparked new expectations for AI-empowered educational transformation. However, it remains unknown which factors affect its effectiveness in empowering learners to solve programming problems. Therefore, this study employed a quasi-experimental research design and used Python graphing in programming education as an example to investigate the factors influencing the effectiveness of learners in applying ChatGPT to problem-solving. Findings: AI literacy significantly influences learners’ effectiveness in using ChatGPT to solve problems, with AI awareness and AI usage being key factors. The knowledge base of programming language significantly affects learners’ effectiveness in applying ChatGPT to solve programming problems. Learners’ cognitive level of ChatGPT significantly influences their effectiveness in applying ChatGPT to problem-solving, while usage intention does not have a significant impact. However, learners’ intention to use ChatGPT significantly improves after application. Based on these findings, this study proposes that in the process of empowering education with Artificial Intelligence Generated Content (AIGC) products, the focus on learners should shift from cultivating their AI usage to AI literacy, laying the foundation for empowering learning with AIGC products. It is suggested to shift from mastering specific knowledge to graph-based rules as a method for empowering learning with AIGC products. Additionally, the focus should shift from enhancing learners’ intention to use the technology to strengthen their technological awareness, thereby creating practical pathways for empowering learning with AIGC products.},
  volume = {11},
  ISSN = {2662-9992},
  url = {http://dx.doi.org/10.1057/s41599-024-02751-w},
  DOI = {10.1057/s41599-024-02751-w},
  number = {1},
  journal = {Humanities and Social Sciences Communications},
  publisher = {Springer Science and Business Media LLC},
  author = {Jing, Yuhui and Wang, Haoming and Chen, Xiaojiao and Wang, Chengliang},
  year = {2024},
  month = Feb,
  critique = {This quasi-experiment has no control group: all n=50 students used ChatGPT, so it can only identify correlates of performance, not ChatGPT's causal effect, despite causal-sounding conclusions throughout. The sample is small and homogeneous (one university; 27 of 50 in educational technology, 10 in computer science, 13 in mechanical engineering), and continuous measures (AI literacy, cognitive level, usage intention) are median-split into arbitrary low/high groups, discarding information and inflating Type I error. Numerous t-tests and ANOVAs are run across many dimensions and subgroups with no multiple-comparison correction, so the few significant results (AI awareness, AI usage) plausibly reflect p-hacking; notably AI evaluation and AI ethics were non-significant, yet the paper still headlines 'AI literacy' as a significant factor. The outcome is a single stacked-bar-chart drawing task scored 0-100 by two raters, a narrow and non-generalizable measure, and the claim that findings transfer to other languages and disciplines is unsupported.}
}

@article{Jost2024,
  title = {The Impact of Large Language Models on Programming Education and Student Learning Outcomes},
  abstract = {Recent advancements in Large Language Models (LLMs) like ChatGPT and Copilot have led to their integration into various educational domains, including software development education. Regular use of LLMs in the learning process is still not well-researched; thus, this paper intends to fill this gap. The paper explores the nuanced impact of informal LLM usage on undergraduate students’ learning outcomes in software development education, focusing on React applications. We carefully designed an experiment involving thirty-two participants over ten weeks where we examined unrestricted but not specifically encouraged LLM use and their correlation with student performance. Our results reveal a significant negative correlation between increased LLM reliance for critical thinking-intensive tasks such as code generation and debugging and lower final grades. Furthermore, a downward trend in final grades is observed with increased average LLM use across all tasks. However, the correlation between the use of LLMs for seeking additional explanations and final grades was not as strong, indicating that LLMs may serve better as a supplementary learning tool. These findings highlight the importance of balancing LLM integration with the cultivation of independent problem-solving skills in programming education.},
  volume = {14},
  ISSN = {2076-3417},
  url = {http://dx.doi.org/10.3390/app14104115},
  DOI = {10.3390/app14104115},
  number = {10},
  journal = {Applied Sciences},
  publisher = {MDPI AG},
  author = {Jošt, Gregor and Taneski, Viktor and Karakatič, Sašo},
  year = {2024},
  month = May,
  pages = {4115},
  critique = {With n=32 students and four one-sided Spearman correlations (average use, code generation, explanations, debugging) run without multiple-comparison correction, the significant results (rho ~ -0.30 to -0.36, p = 0.021-0.045) are weak and fragile. LLM usage is self-reported on a 5-point frequency scale, so measurement error and social-desirability bias are unaddressed, and the design is purely correlational yet repeatedly asserts causal language (e.g., reliance is 'detrimental' to skill development). The authors themselves note that weaker students may simply use LLMs more (reverse causality), but they do not control for prior ability, an obvious confound. There is no randomized control and the 'controlled' phase merely forbids LLMs on the final assignment, so the negative-correlation conclusions are over-claimed.}
}

@inproceedings{Kazemitabaar2024,
  series = {CHI’24},
  title = {CodeAid: Evaluating a Classroom Deployment of an LLM-based Programming Assistant that Balances Student and Educator Needs},
  abstract = {Timely, personalized feedback is essential for students learning programming. LLM-powered tools like ChatGPT offer instant support, but reveal direct answers with code, which may hinder deep conceptual engagement. We developed CodeAid, an LLM-powered programming assistant delivering helpful, technically correct responses, without revealing code solutions. CodeAid answers conceptual questions, generates pseudo-code with line-by-line explanations, and annotates student’s incorrect code with fix suggestions. We deployed CodeAid in a programming class of 700 students for a 12-week semester. A thematic analysis of 8,000 usages of CodeAid was performed, further enriched by weekly surveys, and 22 student interviews. We then interviewed eight programming educators to gain further insights. Our findings reveal four design considerations for future educational AI assistants: D1) exploiting AI’s unique benefits; D2) simplifying query formulation while promoting cognitive engagement; D3) avoiding direct responses while encouraging motivated learning; and D4) maintaining transparency and control for students to asses and steer AI responses.},
  url = {http://dx.doi.org/10.1145/3613904.3642773},
  DOI = {10.1145/3613904.3642773},
  booktitle = {Proceedings of the CHI Conference on Human Factors in Computing Systems},
  publisher = {ACM},
  author = {Kazemitabaar, Majeed and Ye, Runlong and Wang, Xiaoning and Henley, Austin Zachary and Denny, Paul and Craig, Michelle and Grossman, Tovi},
  year = {2024},
  month = May,
  pages = {1–20},
  collection = {CHI'24},
  critique = {Despite its large scale (700 students, ~8,000 usages), this is an observational deployment with no control group and no measurement of learning outcomes, so the four 'design considerations' are grounded in usage logs and perceptions rather than evidence of educational benefit. Only 372 of ~700 students used the tool and just 300 consented, so engagement is self-selected and likely biased toward particular students, and the reported '79\% correct / 86\% helpful' rates come from a human-coded subsample of 1,749 of 8,132 usages. The gender finding (women used it more) rests on a t-test over a self-selected subgroup, and statistics such as Cohen's Kappa are reported selectively. The claim that the assistant avoids revealing direct solutions is undermined by the authors' own finding that Help Write Code generated the exact solution in 53\% of cases. Generalizability is limited to one course/institution, and the educator sample is only eight instructors.}
}

@article{Kosar2024,
  title = {Computer Science Education in ChatGPT Era: Experiences from an Experiment in a Programming Course for Novice Programmers},
  abstract = {The use of large language models with chatbots like ChatGPT has become increasingly popular among students, especially in Computer Science education. However, significant debates exist in the education community on the role of ChatGPT in learning. Therefore, it is critical to understand the potential impact of ChatGPT on the learning, engagement, and overall success of students in classrooms. In this empirical study, we report on a controlled experiment with 182 participants in a first-year undergraduate course on object-oriented programming. Our differential study divided students into two groups, one using ChatGPT and the other not using it for practical programming assignments. The study results showed that the students’ performance is not influenced by ChatGPT usage (no statistical significance between groups with a p-value of 0.730), nor are the grading results of practical assignments (p-value 0.760) and midterm exams (p-value 0.856). Our findings from the controlled experiment suggest that it is safe for novice programmers to use ChatGPT if specific measures and adjustments are adopted in the education process.},
  volume = {12},
  ISSN = {2227-7390},
  url = {http://dx.doi.org/10.3390/math12050629},
  DOI = {10.3390/math12050629},
  number = {5},
  journal = {Mathematics},
  publisher = {MDPI AG},
  author = {Kosar, Tomaž and Ostojić, Dragana and Liu, Yu David and Mernik, Marjan},
  year = {2024},
  month = Feb,
  pages = {629},
  critique = {The experiment reports only null results (p = 0.730-0.856) but interprets this as evidence that ChatGPT use is 'safe,' which is a logical over-claim: absence of a significant difference is not proof of no effect, and no equivalence testing or power analysis is reported. The treatment was weak, as Group I was merely 'encouraged' to use ChatGPT and only 21 of 69 used it for all assignments, so the manipulation may have been too diluted to detect an effect, and the authors post-hoc removed 8 control students who admitted using ChatGPT, introducing selection bias. Group II was not isolated from other online resources, so the control is contaminated, and usage was entirely self-reported. With 27\% missing feedback, attrition, and multiple endpoints tested separately without correction, the conclusions rest on an underpowered, non-equivalence null.}
}

@inproceedings{Koutcheme2024,
  title = {Open Source Language Models Can Provide Feedback: Evaluating LLMs' Ability to Help Students Using GPT-4-As-A-Judge},
  abstract = {Large language models (LLMs) have shown great potential for the automatic generation of feedback in a wide range of computing contexts. However, concerns have been voiced around the privacy and ethical implications of sending student work to proprietary models. This has sparked considerable interest in the use of open source LLMs in education, but the quality of the feedback that such open models can produce remains understudied. This is a concern as providing flawed or misleading generated feedback could be detrimental to student learning. Inspired by recent work that has utilised very powerful LLMs, such as GPT-4, to evaluate the outputs produced by less powerful models, we conduct an automated analysis of the quality of the feedback produced by several open source models using a dataset from an introductory programming course. First, we investigate the viability of employing GPT-4 as an automated evaluator by comparing its evaluations with those of a human expert. We observe that GPT-4 demonstrates a bias toward positively rating feedback while exhibiting moderate agreement with human raters, showcasing its potential as a feedback evaluator. Second, we explore the quality of feedback generated by several leading open-source LLMs by using GPT-4 to evaluate the feedback. We find that some models offer competitive performance with popular proprietary LLMs, such as ChatGPT, indicating opportunities for their responsible use in educational settings.},
  author = {Koutcheme, Charles and Dainese, Nicola and Sarsa, Sami and Hellas, Arto and Leinonen, Juho and Denny, Paul},
  booktitle = {Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1},
  publisher = {ACM},
  year = {2024},
  month = July,
  pages = {52–58},
  DOI = {10.1145/3649217.3653612},
  url = {https://doi.org/10.1145/3649217.3653612},
  critique = {The judge-validation study rests on 150 help requests from a single Dart course with one human expert's annotations as ground truth, and agreement between GPT-4 and that expert is weak-to-moderate (Cohen's kappa 0.48 for completeness, 0.40 for selectivity, and only 0.22 for perceptivity, where an 85\% positive base rate makes the judge barely better than a majority-label dummy). The validation is also circular: GPT-4 was benchmarked only on feedback from its own family (GPT-3.5), and the authors acknowledge a positive, self-serving bias, yet this same possibly-biased judge is then the sole evaluator of the open-source models' feedback in RQ2, with no human verification of those quality scores. Model comparisons rest on single greedy-decoding runs with no statistical tests, confidence intervals, or repeated sampling, so small differences in Figure 3 (e.g., Zephyr-beta at 70\% vs 74\%) are treated as meaningful without evidence. Claims that open-source models are competitive with proprietary ones therefore inherit all of the judge's measurement error and bias, on a one-institution, one-language (Dart) dataset.}
}

@article{Li2025,
  title = {Generative artificial intelligence-supported programming education: Effects on learning performance, self-efficacy and processes},
  abstract = {Recent advancements in generative artificial intelligence (GenAI) have drawn significant attention from educators and researchers. However, its effects on learners’ programming performance, self-efficacy and learning processes remain inconclusive, while the mechanisms underlying its efficiency-enhancing potential are underexplored. This study addresses these gaps through a quasi-experiment comparing an experimental group using GenAI for self-directed programming learning with a control group relying on alternative tools. Additionally, the experimental group was divided into high- and low-performance subgroups to examine the relationship between learning behaviour patterns and academic outcomes using process mining techniques. The findings reveal that (a) GenAI demonstrates strong potential to enhance learning outcomes and self-efficacy but negatively affects long-term knowledge transfer; (b) excessive reliance on GenAI and cognitive outsourcing impede effective knowledge acquisition; (c) high-performing learners exhibit greater epistemic agency, actively critiquing and engaging with AI-generated content to construct knowledge proactively. This study underscores the risks of over-reliance on GenAI and the detrimental effects of cognitive offloading, highlighting the critical role of cognitive engagement and epistemic agency in fostering hybrid intelligence. It provides empirical and theoretical insights to inform the design of GenAI-supported programming education strategies and interventions. Implications for practice or policy: Instructors can enhance programming self-efficacy by integrating GenAI tools like ChatGPT into self-learning activities, particularly for reinforcing academic performance. Course leaders should emphasise GenAI use in programming courses to support student engagement, though they should also prepare students for problem-solving without external resources. Educational institutions may consider developing guidelines for balanced GenAI usage to maximise learning benefits while addressing potential limitations in problem-solving skills.},
  ISSN = {1449-3098},
  url = {http://dx.doi.org/10.14742/ajet.9932},
  DOI = {10.14742/ajet.9932},
  journal = {Australasian Journal of Educational Technology},
  publisher = {Australasian Society for Computers in Learning in Tertiary Education},
  author = {Li, Siran and Liu, Jiangyue and Dong, Qianyan},
  year = {2025},
  month = May,
  critique = {The quasi-experiment involves only 55 physics-education undergraduates at a single Chinese university (27 vs 28 per group) in one 120-minute session, and the comparison confounds the intervention itself: the experimental group used GPT-3.5-turbo while the control group used 'other Internet resources', so differences may reflect instant answer generation versus web search rather than GenAI per se. The 'negatively affects long-term knowledge transfer' conclusion partly rests on a null result (transfer test Z = -0.615, p = 0.538) interpreted as evidence of no transfer, while the significant transfer-phase deficits in logical correctness (p = 0.025) and functionality (p = 0.020) come from a large battery of unadjusted tests (6 rubric dimensions x 2 phases, 3 self-efficacy dimensions, 15 behavior codes) with no multiple-comparison correction. The subgroup analysis clusters the 27 experimental students into groups of 13 and 14 and then tests those clusters for posttest-score differences (p = 0.039), a circular, underpowered procedure, and the behavior-pattern differences are narrated as causal mechanisms (cognitive offloading, metacognitive laziness) the correlational data cannot support. Self-efficacy effects rely on self-report, and Table 9's 'paired samples t test' label does not match the independent-samples tests actually shown.}
}

@inproceedings{Liu2024,
  title = {Can Small Language Models With Retrieval-Augmented Generation Replace Large Language Models When Learning Computer Science?},
  abstract = {Leveraging Large Language Models (LLMs) for personalized learning and support is becoming a promising tool in computing education. AI Assistants can help students with programming, problem-solving, converse with them to clarify course content, explain error messages to help with debugging, and much more. However, using cloud-based LLMs poses risks around data security, privacy, but also control of the overarching system. To address these concerns, we created a locally-stored Small Language Model (SLM) that leverages different Retrieval-Augmented Generation (RAG) methods to support computing students' learning. We compare one SLM (neural-chat-7b-v3 - fine-tuned version of Mistral-7B-v0.1) against two popular LLMs (gpt-3.5-turbo and gpt-4-32k) to see the viability for computing educators to use in their course(s). We use conversations from a CS1 course (N = 1,260), providing students with an AI Assistant (using gpt-3.5-turbo) to help them learn content and support problem-solving while completing their Python programming assignment. In total, we had 269 students use the AI Assistant, with a total of 1,988 questions asked. Using this real conversational data, we re-ran student questions using our novel SLM (neural-chat-7b-v3 testing nine different RAG methods) and gpt-4-32k, then compared those results against the original gpt-3.5-turbo responses. Our findings indicate that using an SLM with RAG can perform similarly, if not better, than LLMs. This shows that it is possible for computing educators to use SLMs (with RAG) in their course(s) as a tool for scalable learning, supporting content understanding and problem-solving needs, while employing their own policies on data privacy and security.},
  author = {Liu, Suqing and Yu, Zezhu and Huang, Feiran and Bulbulia, Yousef and Bergen, Andreas and Liut, Michael},
  booktitle = {Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1},
  publisher = {ACM},
  year = {2024},
  month = July,
  pages = {388–393},
  DOI = {10.1145/3649217.3653554},
  url = {https://doi.org/10.1145/3649217.3653554},
  critique = {Answer quality was scored only by LLMs (gpt-3.5-turbo and gpt-4-32k rating every model including themselves on a 0-100 scale) with no human expert evaluation, making the central quality metric self-referential and prone to self-preference: the same gpt-4 responses score 85.8 under the 'OpenAI' rubric but 70.85 under the 'open source' rubric. No statistical analysis is reported (no N for how many of the 1,988 questions were scored, no confidence intervals, no significance tests), and the headline recommendation to replace LLMs is contradicted by the paper's own Table 1, where gpt-3.5-turbo and gpt-4-32k outperform neural-chat-7b-v3 under both rubrics; the 'surpassing gpt-4' claim (71.55 vs 70.85) is a 0.7-point gap from one prototype with no variance information. There is no student or learning-outcome evaluation (the authors concede the lack of deployment and direct student feedback), yet the discussion recommends the system for CS1 students, and generalizability is limited to one course, one institution, and a single SLM.}
}

@inbook{Ma2024,
  title = {Enhancing Programming Education with ChatGPT: A Case Study on Student Perceptions and Interactions in a Python Course},
  abstract = {The integration of ChatGPT as a supportive tool in education, notably in programming courses, addresses the unique challenges of programming education by providing assistance with debugging, code generation, and explanations. Despite existing research validating ChatGPT's effectiveness, its application in university-level programming education and a detailed understanding of student interactions and perspectives remain limited. This paper explores ChatGPT's impact on learning in a Python programming course tailored for first-year students over eight weeks. By analyzing responses from surveys, open-ended questions, and student-ChatGPT dialog data, we aim to provide a comprehensive view of ChatGPT's utility and identify both its advantages and limitations as perceived by students. Our study uncovers a generally positive reception toward ChatGPT and offers insights into its role in enhancing the programming education experience. These findings contribute to the broader discourse on AI's potential in education, suggesting paths for future research and application.},
  ISBN = {9783031643156},
  ISSN = {1865-0937},
  url = {http://dx.doi.org/10.1007/978-3-031-64315-6_9},
  DOI = {10.1007/978-3-031-64315-6_9},
  booktitle = {Artificial Intelligence in Education. Posters and Late Breaking Results, Workshops and Tutorials, Industry and Innovation Tracks, Practitioners, Doctoral Consortium and Blue Sky},
  publisher = {Springer Nature Switzerland},
  author = {Ma, Boxuan and Chen, Li and Konomi, Shin’ichi},
  year = {2024},
  pages = {113–126},
  critique = {The learning-evaluation study has 19 paid volunteers in a single-arm pre-post design with no control group, so the 12\% score gain (p = 0.033) and 14\% time reduction (p = 0.003) cannot be separated from practice effects, repeated testing, or novelty; worse, the first problem used inside the intervention was the same problem as the pre-test, directly inflating post-test gains. Multiple significance tests are run on this tiny sample (overall and per-objective scores and times, confidence, engagement ratings) without correction, and the LO1 gain is only 6.1\% with the authors themselves noting an item-difficulty confound (pre-test means 54\% vs 38\%). The 'fourfold efficiency' claim compares LLM generation against exactly two human TAs each producing materials for one problem (a near-anecdotal baseline), and the 90\% success figure rests on the authors' own annotation codebook with one author rating the remaining materials, while raw buggy-code generation succeeded only 57.9\% of the time. The abstract's 'significantly improves student performance' and 'outperforming human counterparts fourfold' overstate what a small, self-selected, lab-based feasibility study can support.}
}

@inproceedings{Padurean2026,
  title = {Interleaving Natural Language Prompting with Code Editing for Solving Programming Tasks with Generative AI Models},
  abstract = {Modern computing students often rely on both natural-language prompting and manual code editing to solve programming tasks. Yet we still lack a clear understanding of how these two modes are combined in practice, and how their usage varies with task complexity and student ability. In this paper, we investigate this through a large-scale study in an introductory programming course, collecting 13,305 interactions from 355 students during a three-day lab activity. Our analysis shows that students primarily use prompting to generate initial solutions, and then often enter short edit-run loops to refine their code following a failed execution. Student reflections confirm that prompting is helpful for structuring solutions, editing is effective for making targeted corrections, while both are useful for learning. We find that manual editing becomes more frequent as task complexity increases, but most edits remain concise, with many affecting a single line of code. Higher-performing students succeed with less reliance on editing and fewer overall interactions. These findings highlight the role of manual editing as a form of last-mile repair, complementing prompting in AI-assisted programming workflows.},
  author = {Pădurean, Victor-Alexandru and Gotovos, Alkis and Ghosh, Ahana and Denny, Paul and Leinonen, Juho and Luxton-Reilly, Andrew and Prather, James and Singla, Adish},
  booktitle = {Proceedings of the 31st ACM Conference on Innovation and Technology in Computer Science Education V. 1},
  publisher = {ACM},
  year = {2026},
  month = July,
  pages = {273–279},
  DOI = {10.1145/3803400.3809312},
  url = {https://doi.org/10.1145/3803400.3809312},
  critique = {This is an observational, single-course study (355 students, 13,305 interactions over a three-day lab) with no control condition and no causal identification, so statements that editing 'complements' prompting are descriptive associations rather than established effects. 'Higher-performing students' and 'task complexity' are defined by the authors' own proxies, and learning benefits are partly inferred from student reflections, which are self-report and subject to social-desirability bias. The three-day window, one institution, and one course limit generalizability, and the frequency analyses do not demonstrate that the observed edit-run loops actually improve understanding or transfer.}
}

@inproceedings{Pankiewicz2024,
  series = {ITiCSE'24},
  title = {Navigating Compiler Errors with AI Assistance - A Study of GPT Hints in an Introductory Programming Course},
  abstract = {We examined the efficacy of AI-assisted learning in an introductory programming course at the university level by using a GPT-4 model to generate personalized hints for compiler errors within a platform for automated assessment of programming assignments. The control group had no access to GPT hints. In the experimental condition GPT hints were provided when a compiler error was detected, for the first half of the problems in each module. For the latter half of the module, hints were disabled. Students highly rated the usefulness of GPT hints. In affect surveys, the experimental group reported significantly higher levels of focus and lower levels of confrustion (confusion and frustration) than the control group. For the six most commonly occurring error types we observed mixed results in terms of performance when access to GPT hints was enabled for the experimental group. However, in the absence of GPT hints, the experimental group's performance surpassed the control group for five out of the six error types.},
  url = {http://dx.doi.org/10.1145/3649217.3653608},
  DOI = {10.1145/3649217.3653608},
  booktitle = {Proceedings of the 2024 on Innovation and Technology in Computer Science Education V. 1},
  publisher = {ACM},
  author = {Pankiewicz, Maciej and Baker, Ryan S.},
  year = {2024},
  month = July,
  pages = {94–100},
  collection = {ITiCSE 2024},
  critique = {The study is small and confined to one course and one language (C#, hints in Polish), and the authors concede the limited sample and single-institution setting. The within-module A/B design, with hints enabled for the first half of each module and disabled for the second, is confounded by order and carryover: the experimental group's superior no-hint performance in the second half may reflect learning or practice from having had hints earlier, not a durable 'lasting skills' effect, yet it is presented as a substantive finding. Results with hints enabled were mixed (performance was even lower for some error types), and the affect findings rest on a small, self-reported survey sample. The claim that hints 'equip students with lasting skills' is a post-hoc reinterpretation of an unexpected pattern.}
}

@inproceedings{Prather2024,
  title = {The Widening Gap: The Benefits and Harms of Generative AI for Novice Programmers},
  abstract = {Novice programmers often struggle through programming problem solving due to a lack of metacognitive awareness and strategies. Previous research has shown that novices can encounter multiple metacognitive difficulties while programming, such as forming incorrect conceptual models of the problem or having a false sense of progress after testing their solution. Novices are typically unaware of how these difficulties are hindering their progress. Meanwhile, many novices are now programming with generative AI (GenAI), which can provide complete solutions to most introductory programming problems, code suggestions, hints for next steps when stuck, and explain cryptic error messages. Its impact on novice metacognition has only started to be explored. Here we replicate a previous study that examined novice programming problem solving behavior and extend it by incorporating GenAI tools. Through 21 lab sessions consisting of participant observation, interview, and eye tracking, we explore how novices are coding with GenAI tools. Although 20 of 21 students completed the assigned programming problem, our findings show an unfortunate divide in the use of GenAI tools between students who did and did not struggle. Some students who did not struggle were able to use GenAI to accelerate, creating code they already intended to make, and were able to ignore unhelpful or incorrect inline code suggestions. But for students who struggled, our findings indicate that previously known metacognitive difficulties persist, and that GenAI unfortunately can compound them and even introduce new metacognitive difficulties. Furthermore, struggling students often expressed cognitive dissonance about their problem solving ability, thought they performed better than they did, and finished with an illusion of competence. Based on our observations from both groups, we propose ways to scaffold the novice GenAI experience and make suggestions for future work.},
  author = {Prather, James and Reeves, Brent N. and Leinonen, Juho and MacNeil, Stephen and Randrianasolo, Arisoa S. and Becker, Brett A. and Kimmel, Bailey and Wright, Jared and Briggs, Ben},
  booktitle = {Proceedings of the 2024 ACM Conference on International Computing Education Research - Volume 1},
  publisher = {ACM},
  year = {2024},
  month = Aug,
  pages = {469–486},
  DOI = {10.1145/3632620.3671116},
  url = {https://doi.org/10.1145/3632620.3671116},
  critique = {This is a small (N=21), single-institution observational study using participant observation, interviews, and eye tracking, with no control condition, so the strong 'widening gap' narrative rests on 21 students and qualitative interpretation. Students self-selected into lab sessions, and the division into 'struggling' versus 'not struggling' is the authors' own classification, so the claim that GenAI compounds metacognitive difficulties for weaker students is vulnerable to confirmation bias and cannot be causally attributed to GenAI. 'Illusion of competence' is inferred from think-aloud and post-hoc interviews rather than measured, and the study reports no objective learning or transfer outcome.}
}

@inproceedings{Ramachandra2026,
  title = {Detecting AI-Generated Code in Introductory Programming Courses},
  abstract = {With the rapid surge of generative AI, many tools have been introduced, such as Google's Gemini and OpenAI's GPT-4, with the well-intentioned goal of supporting programmers [5,8]. These tools can be used by professional programmers to help write code efficiently as well as support debugging and testing; however, we recently began to notice an increase in the number of novice programmers who have become highly dependent on Large Language Models (LLMs) to code for them rather than using LLMs as a learning tool [2,10]. In our CS1 course, approximately 10-15\% of the students (out of ~350) were cited for academic misconduct due to direct plagiarism from LLMs, many of which performed poorly due to an over-reliance on generative AI. To address this, we developed a machine learning-based tool to detect AI-generated code. The tool utilized datasets consisting of thousands of student submissions (in C++) from introductory programming courses and we created an equal number of AI-generated solutions using carefully curated prompts. We trained traditional ML models (Random Forest, XGBoost, etc.) on a labeled datasets, and our best-performing model achieved high (>98\%) precision and recall. Our work is unique as we are able to demonstrate that the models remained robust even when trained with noisy data (including AI-generated submissions in student-written dataset). Our goal is to provide the community with a model that can be customized to any course program to encourage early detection and intervention of plagiarized code generated by LLMs.},
  author = {Ramachandra, Aryan and Chaudhary, Suhani and Tran, Justin and Desai, Riti and Pang, Ashley and Salloum, Mariam},
  booktitle = {Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1},
  publisher = {ACM},
  year = {2026},
  month = Feb,
  pages = {894–900},
  DOI = {10.1145/3770762.3772522},
  url = {https://doi.org/10.1145/3770762.3772522},
  critique = {Label validity is the core weakness: the 'student-written' negative class comes from zyBooks submissions assumed (not verified) to be authentic, and the authors simulate poisoning by injecting their own AI samples rather than measuring real contamination, so the true false-positive risk against genuinely AI-assisted student work is unknown. The AI-positive class is generated by only two models (GPT-4o and Gemini 2.5 Pro), so the detector is really a 'these-two-models' classifier and the authors admit it may miss Claude and paraphrased/rewritten output. The near-perfect metrics (F1 0.995 for CS1, 0.988 for CS3) signal trivially separable stylistic artifacts rather than robust AI-authorship detection, and with 10 prompt variations per problem there is no reported cross-split deduplication, so near-duplicate AI samples may leak between train and test and inflate scores. Sampling to equal class sizes (280/lab CS1, 100/lab CS3) also discards the real-world class imbalance the tool would face, and the study is confined to one institution, C++, and a small set of short labs.},
}

@inproceedings{Sheese2024,
  series = {ACE'24},
  title = {Patterns of Student Help-Seeking When Using a Large Language Model-Powered Programming Assistant},
  abstract = {Providing personalized assistance at scale is a long-standing challenge for computing educators, but a new generation of tools powered by large language models (LLMs) offers immense promise. Such tools can, in theory, provide on-demand help in large class settings and be configured with appropriate guardrails to prevent misuse and mitigate common concerns around learner over-reliance. However, the deployment of LLM-powered tools in authentic classroom settings is still rare, and very little is currently known about how students will use them in practice and what type of help they will seek. To address this, we examine students’ use of an innovative LLM-powered tool that provides on-demand programming assistance without revealing solutions directly. We deployed the tool for 12 weeks in an introductory computer and data science course (n = 52), collecting more than 2,500 queries submitted by students throughout the term. We manually categorized all student queries based on the type of assistance sought, and we automatically analyzed several additional query characteristics. We found that most queries requested immediate help with programming assignments, whereas fewer requests asked for help on related concepts or for deepening conceptual understanding. Furthermore, students often provided minimal information to the tool, suggesting this is an area in which targeted instruction would be beneficial. We also found that students who achieved more success in the course tended to have used the tool more frequently overall. Lessons from this research can be leveraged by programming educators and institutions who plan to augment their teaching with emerging LLM-powered tools.},
  url = {http://dx.doi.org/10.1145/3636243.3636249},
  DOI = {10.1145/3636243.3636249},
  booktitle = {Proceedings of the 26th Australasian Computing Education Conference},
  publisher = {ACM},
  author = {Sheese, Brad and Liffiton, Mark and Savelka, Jaromir and Denny, Paul},
  year = {2024},
  month = Jan,
  pages = {49–57},
  collection = {ACE 2024},
  critique = {The study is a single-course, single-institution observational deployment (n=52, n=49 analyzed, taught by one author) with no control or comparison group. Tool use was entirely self-selected, so the reported positive correlation between usage and course performance (r=0.38, p=0.013) is confounded by motivation/engagement and cannot support the 'no evidence of harm' conclusion the discussion leans toward, a point the authors partially concede. Only the queries students voluntarily submitted were analyzed, with no measurement of whether responses improved learning or of pre/post skill change. Query categorization was done subjectively by the authors (Cohen's kappa 0.75), the 'low-effort' thresholds (issue <10 characters, >80\% copied) are arbitrary, and one extreme outlier (614 queries) was excluded from the correlational analyses, a defensible but under-justified data-handling choice.}
}

@inproceedings{Shihab2025,
  title = {The Effects of GitHub Copilot on Computing Students' Programming Effectiveness, Efficiency, and Processes in Brownfield Coding Tasks},
  abstract = {When graduates of computing degree programs enter the software industry, they will most likely join teams working on legacy code bases developed by people other than themselves. In these so-called brownfield software development settings, generative artificial intelligence (GenAI) coding assistants like GitHub Copilot are rapidly transforming software development practices, yet the impact of GenAI on student programmers performing brownfield development tasks remains underexplored. This paper investigates how GitHub Copilot influences undergraduate students’ programming performance, behaviors, and understanding when completing brownfield programming tasks in which they add new code to an unfamiliar code base. We conducted a controlled experiment in which 10 undergraduate computer science students completed highly similar brownfield development tasks with and without Copilot in a legacy web application. Using a mixed-methods approach combining performance analysis, behavioral analysis, and exit interviews, we found that students completed tasks 34.9\% faster (p < 0.05) and made 50.0\% more solution progress (p < 0.05) when using Copilot. Moreover, our analysis revealed that, when using Copilot, students spent 10.6\% less time manually writing code (p < 0.05), and 11.6\% less time conducting web searches (p < 0.05), providing evidence of a fundamental shift in how they engaged in programming. In exit interviews, students reported concerns about not understanding how or why Copilot suggestions work. This research suggests the need for computing educators to develop new pedagogical approaches that leverage GenAI assistants’ benefits while fostering reflection on how and why GenAI suggestions address brownfield programming tasks. Complete study results and analysis are presented at ghcopilot-icer.github.io.},
  author = {Shihab, Md Istiak Hossain and Hundhausen, Christopher and Tariq, Ahsun and Haque, Summit and Qiao, Yunhan and Mulanda, Brian Wise},
  booktitle = {Proceedings of the 2025 ACM Conference on International Computing Education Research V.1},
  publisher = {ACM},
  year = {2025},
  month = Aug,
  pages = {407–420},
  DOI = {10.1145/3702652.3744219},
  url = {https://doi.org/10.1145/3702652.3744219},
  critique = {The sample is only n=10 from a single university, deliberately restricted to high-performing students (A/A- in prior web development, prior project experience, 3rd/4th year, screened to have little/no GenAI experience), so the 'first-ever empirical account' claim and the 34.9\% speedup are unlikely to generalize. Treatment order was not counterbalanced (everyone did No-Copilot first), so the efficiency gain is confounded with practice and growing familiarity with the codebase and environment across the two tasks, which the authors acknowledge but cannot remove. The time analysis is limited to Task 1 (the only task all participants completed), and the two tasks, though matched on LOC/statements, are not identical. With n=10 the study runs many statistical tests (permutation chi-squares, multiple Wilcoxon signed-rank tests across ~10 activity categories) where some 'significant' differences are tautological because Copilot-only categories (write prompt, view response) cannot occur without Copilot, and the higher/lower-performer split (4 vs 6) yields a single significant residual difference (pasting Copilot code) that is over-interpreted.}
}

@article{Sun2024,
  title = {Would ChatGPT-facilitated programming mode impact college students’ programming behaviors, performances, and perceptions? An empirical study},
  abstract = {ChatGPT, an AI-based chatbot with automatic code generation abilities, has shown its promise in improving the quality of programming education by providing learners with opportunities to better understand the principles of programming. However, limited empirical studies have explored the impact of ChatGPT on learners’ programming processes. This study employed a quasi-experimental design to explore the possible impact of ChatGPT-facilitated programming mode on college students’ programming behaviors, performances, and perceptions. 82 college students were randomly divided into two classes. One class employed ChatGPT-facilitated programming (CFP) practice and the other class utilized self-directed programming (SDP) mode. Mixed methods were utilized to collect multidimensional data. Data analysis uncovered some intriguing results. Firstly, students in the CFP mode had more frequent behaviors of debugging and receiving error messages, as well as pasting console messages on the website and reading feedback. At the same time, students in the CFP mode had more frequent behaviors of copying and pasting codes from ChatGPT and debugging, as well as pasting codes to ChatGPT and reading feedback from ChatGPT. Secondly, CFP practice would improve college students’ programming performance, while the results indicated that there was no statistically significant difference between the students in CFP mode and the SDP mode. Thirdly, student interviews revealed three highly concerned themes from students' user experience about ChatGPT: the services offered by ChatGPT, the stages of ChatGPT usage, and experience with ChatGPT. Finally, college students’ perceptions toward ChatGPT significantly changed after CFP practice, including its perceived usefulness, perceived ease of use, and intention to use. Based on these findings, the study proposes implications for future instructional design and the development of AI-powered tools like ChatGPT.},
  volume = {21},
  ISSN = {2365-9440},
  url = {http://dx.doi.org/10.1186/s41239-024-00446-5},
  DOI = {10.1186/s41239-024-00446-5},
  number = {1},
  journal = {International Journal of Educational Technology in Higher Education},
  publisher = {Springer Science and Business Media LLC},
  author = {Sun, Dan and Boudouaia, Azzeddine and Zhu, Chengcong and Li, Yan},
  year = {2024},
  month = Feb,
  critique = {The abstract claims 82 students were 'randomly divided' into two classes, but the methods describe a quasi-experiment on two intact classes (43 CFP vs 39 SDP) at one Chinese university, and the paper contains implausible internal inconsistencies that undermine trust: the study is dated 'spring term of 2020' (before ChatGPT existed), the instructor is identified as both the first and the fourth author, and the reported gender counts do not sum to the stated group sizes (19F+13M=32, not 43). Programming performance was compared with a posttest-only t-test on final projects with no pretest or baseline equivalence check, and the null result (t=1.28, p=0.204) is contradicted by the conclusion's claim that ChatGPT 'significantly enhances students' programming studies'. The perception findings come from four separate pre-post t-tests within the CFP group only, with no control-group comparison and no multiple-comparison correction, and the lag-sequential behavioral analysis is purely descriptive, so it cannot support causal claims. A five-session, single-course, single-instructor design with students unaware of the different treatments further limits generalizability.}
}

@inproceedings{Yang2024,
  title = {Debugging with an AI Tutor: Investigating Novice Help-seeking Behaviors and Perceived Learning},
  abstract = {Debugging is a crucial skill for programmers, yet it can be challenging for novices to learn. The introduction of large language models (LLMs) has opened up new possibilities for providing personalized debugging support to students. However, concerns have been raised about potential student over-reliance on LLM-based tools. This mixed-methods study investigates how a pedagogically-designed LLM-based chatbot supports students’ debugging efforts in an introductory programming course. We conducted interviews and debugging think-aloud tasks with 20 students at three points throughout the semester. We specifically focused on characterizing when students initiate help from the chatbot during debugging, how they engage with the chatbot’s responses, and how they describe their learning experiences with the chatbot. By analyzing data from the debugging tasks, we identified varying help-seeking behaviors and levels of engagement with the chatbot’s responses, depending on students’ familiarity with the suggested strategies. Interviews revealed that students appreciated the content and experiential knowledge provided by the chatbot, but did not view it as a primary source for learning debugging strategies. Additionally, students self-identified certain chatbot usage behaviors as negative, “non-ideal” engagement and others as positive, “learning-oriented” usage. Based on our findings, we discuss pedagogical implications and future directions for designing pedagogical chatbots to support debugging.},
  author = {Yang, Stephanie and Zhao, Hanzhang and Xu, Yudian and Brennan, Karen and Schneider, Bertrand},
  booktitle = {Proceedings of the 2024 ACM Conference on International Computing Education Research - Volume 1},
  publisher = {ACM},
  year = {2024},
  month = Aug,
  pages = {84–94},
  DOI = {10.1145/3632620.3671092},
  url = {https://doi.org/10.1145/3632620.3671092},
  critique = {This qualitative study of 20 compensated volunteers from a single elite course (Harvard's CS50) has no control group and measures no learning outcomes, so all 'perceived learning' conclusions rest on self-report interviews and observed think-alouds; the authors themselves concede the need for objective performance measures. Key claims rest on very small cell counts - only 2 hypothesis-testing prompts total, 12 chatbot 'strategy' responses (5 used, 7 ignored), and 2 incorrect chatbot solutions - making frequency-based generalizations fragile. Recruitment was self-selective, all interviews were conducted by the first author (interviewer bias), and the presence of a researcher during think-aloud tasks plus the use of pre-written rather than students' own code likely distorts naturalistic behavior (both acknowledged). Inter-rater reliability (Fleiss' kappa) was computed on only 20\% of the data before individual coding, and the findings are specific to one chatbot whose RAG was grounded in CS50's own lecture captions, limiting generalizability to other courses and tools.}
}

@article{Yang2025,
  title = {The effectiveness of ChatGPT in assisting high school students in programming learning: evidence from a quasi-experimental research},
  abstract = {Programming education gains importance in high schools as the digital age progresses. However, the openness and adaptability of programming languages present unique challenges for instructional practices compared to other subjects. While traditional instructional tools offer limited support, ChatGPT, a groundbreaking Generative Artificial Intelligence, has shown impressive capabilities in natural language processing and knowledge generation. This study explored whether ChatGPT can transcend existing limitations and improve programming education through a quasi-experimental approach with post-hoc interviews in high school classrooms. A total of 153 students participated, and the results from MANCOVA and ANCOVA analyses revealed that students using ChatGPT reported lower levels of flow experience, self-efficacy, and learning achievement compared to those utilizing conventional methods. Post-hoc interviews further revealed that students felt ChatGPT's effectiveness in facilitating their programming learning fell short of their initial expectations. These findings highlight the need to carefully consider the complexity of programming learning tasks and students' cognitive, affective, and interactive dimensions when integrating AI technologies into education. We discuss the implications and provide thorough pedagogical strategies, specifically the guidance-practice-transformation (G-P-T) mode, to maximize the potential of AI tools and support high school programming education, emphasizing the balance of technological innovation with learning best practices.},
  volume = {33},
  ISSN = {1744-5191},
  url = {http://dx.doi.org/10.1080/10494820.2025.2450659},
  DOI = {10.1080/10494820.2025.2450659},
  number = {6},
  journal = {Interactive Learning Environments},
  publisher = {Informa UK Limited},
  author = {Yang, Tzu-Chi and Hsu, Yi-Chuan and Wu, Jiun-Yu},
  year = {2025},
  month = Jan,
  pages = {3726–3743},
  critique = {Despite claiming random assignment, the 153 students (all girls, from one Taiwanese high school) were assigned at class level (three classes per group), yet all analyses (MANCOVA/ANCOVA, df=148) treat students as independent, ignoring cluster nesting; the paper also swaps the group sizes between text (experimental n=77) and Table 1 (control n=77), and the MANCOVA univariate F values (28.32, 53.0, 4.36) are inconsistent with the follow-up ANCOVA values (9.20, 15.78, 3.96). Both groups' achievement collapsed from roughly 80 to 50/42 on the posttest, so the borderline achievement effect (F=3.96, p=0.048, partial eta-squared=0.03) rests on a small difference on a much harder test, with no correction for the three outcomes tested. Treatment fidelity was unchecked: there are no usage logs, no measure of prior ChatGPT experience, and control students could use ChatGPT outside class, so the three-week intervention difference is confounded with attention and novelty. The interviews come from only 11 self-selected experimental-group volunteers, and the single-sex, single-school, single-teacher setting severely limits generalizability.}
}

@article{Ye2025,
  title = {Improving students’ programming performance: an integrated mind mapping and generative AI chatbot learning approach},
  abstract = {With the development of the times, programming education has become increasingly important for individual development. However, for programming beginners such as primary and secondary school students, learning programming is not a simple task and requires additional learning support. Generative AI (GenAI) chatbots are effective teaching aids that can reduce the learning difficulty of programming by providing real-time guidance and personalized learning support based on students’ abilities. Therefore, it has been a trend to apply GenAI chatbots in teaching. However, technology always has two sides. Over-reliance on these chatbots may weaken students’ ability to think independently and affect their learning effectiveness. Therefore, how to rationally utilize GenAI chatbots in the classroom and improve their application effectiveness has become an important issue for both researchers and frontline teachers. Based on this, the present study proposed a learning method that integrates mind mapping with GenAI chatbots. To assess the effectiveness of this learning method and to investigate whether there are differences in the impact of various types of mind mapping-supported GenAI chatbots on students’ programming academic performance, computational thinking, and programming self-efficacy, the research team conducted a quasi-experimental study. The participants were 111 seventh-grade students at a junior high school in southeastern China. Experimental Group 1 (36 students) used a learning approach that integrated progressive mind maps with a Generative AI chatbot, Experimental Group 2 (36 students) used a learning approach that integrated self-constructed mind maps with a GenAI chatbot, and the control group (39 students) used a traditional AI chatbot-based learning approach. The results showed that participants in both experimental groups had significantly better programming learning performance as well as computational thinking than the control group, and that the learning method integrating progressive mind mapping with GenAI chatbots was more effective.},
  volume = {12},
  ISSN = {2662-9992},
  url = {http://dx.doi.org/10.1057/s41599-025-04846-4},
  DOI = {10.1057/s41599-025-04846-4},
  number = {1},
  journal = {Humanities and Social Sciences Communications},
  publisher = {Springer Science and Business Media LLC},
  author = {Ye, Xindong and Zhang, Wenyu and Zhou, Yuxin and Li, Xiaozhi and Zhou, Qiang},
  year = {2025},
  month = Apr,
  critique = {The 111 seventh-graders were assigned by intact class (two classes to the interventions, one to control - only three clusters), yet every analysis treats students as independent (df=108) with no cluster adjustment, and the 'randomly selected' classes are never actually described as randomized. With roughly ten outcome tests (achievement, CT total plus five dimensions, self-efficacy) and uncorrected Tukey post-hocs, several borderline p-values (0.013-0.047) are plausibly false positives; the self-efficacy result is the clearest over-claim: the posttest ANOVA was null (F=1.39, p=0.253) and all three groups - including the chatbot-only control - improved on paired t-tests, yet the discussion and conclusion credit the mind-mapping integration with improving self-efficacy. The control group also used the same GenAI chatbot, so the design isolates only the mind-mapping scaffold, not any chatbot effect, and the promised fidelity data (no interaction logs) and interviews are never analyzed. Other red flags include a Welch-corrected df (71.8) appearing without explanation in the achievement ANOVA, a sign/mean typo in Table 15, an 11-week versus 6-week duration inconsistency, and a 5-point (out of 100) mean difference supporting the headline superiority claim.}
}